From f8e1b603cf79140695b21162a624c06d193cded5 Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 20:30:23 -0300 Subject: [PATCH 1/8] clean release.yaml --- release.yaml | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/release.yaml b/release.yaml index c1ec4c1e12..8b13789179 100644 --- a/release.yaml +++ b/release.yaml @@ -1,22 +1 @@ -fleet: - - 104.1.5+up0.10.9 -fleet-agent: - - 104.1.5+up0.10.9 -fleet-crd: - - 104.1.5+up0.10.9 -harvester-csi-driver: - - 104.0.5+up0.1.23 -rancher-logging: - - 104.2.0+up4.8.0 -rancher-logging-crd: - - 104.2.0+up4.8.0 -rancher-monitoring: - - 104.1.4+up57.0.3 -rancher-monitoring-crd: - - 104.1.4+up57.0.3 -rancher-webhook: - - 104.0.7+up0.5.7 -rancher-cis-benchmark: - - 6.7.0 -rancher-cis-benchmark-crd: - - 6.7.0 + From 00611ea14efa7e25dda9a0a06ccc96d982e1ed6e Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 20:31:11 -0300 Subject: [PATCH 2/8] fp: fleet 103.1.13+up0.9.15 --- assets/fleet/fleet-103.1.13+up0.9.15.tgz | Bin 0 -> 5344 bytes charts/fleet/103.1.13+up0.9.15/Chart.yaml | 22 ++++ charts/fleet/103.1.13+up0.9.15/README.md | 30 +++++ .../charts/gitjob/.helmignore | 23 ++++ .../charts/gitjob/Chart.yaml | 5 + .../charts/gitjob/templates/_helpers.tpl | 7 ++ .../charts/gitjob/templates/clusterrole.yaml | 38 ++++++ .../gitjob/templates/clusterrolebinding.yaml | 12 ++ .../charts/gitjob/templates/deployment.yaml | 52 ++++++++ .../charts/gitjob/templates/leases.yaml | 23 ++++ .../charts/gitjob/templates/service.yaml | 12 ++ .../gitjob/templates/serviceaccount.yaml | 4 + .../charts/gitjob/values.yaml | 27 +++++ .../103.1.13+up0.9.15/templates/_helpers.tpl | 22 ++++ .../templates/configmap.yaml | 26 ++++ .../templates/deployment.yaml | 102 ++++++++++++++++ .../job_cleanup_clusterregistrations.yaml | 40 ++++++ .../103.1.13+up0.9.15/templates/rbac.yaml | 114 ++++++++++++++++++ .../templates/serviceaccount.yaml | 12 ++ charts/fleet/103.1.13+up0.9.15/values.yaml | 87 +++++++++++++ index.yaml | 26 ++++ release.yaml | 3 +- 22 files changed, 686 insertions(+), 1 deletion(-) create mode 100644 assets/fleet/fleet-103.1.13+up0.9.15.tgz create mode 100644 charts/fleet/103.1.13+up0.9.15/Chart.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/README.md create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/.helmignore create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/Chart.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/_helpers.tpl create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrole.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrolebinding.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/deployment.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/leases.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/service.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/serviceaccount.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/values.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/templates/_helpers.tpl create mode 100644 charts/fleet/103.1.13+up0.9.15/templates/configmap.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/templates/deployment.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/templates/job_cleanup_clusterregistrations.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/templates/rbac.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/templates/serviceaccount.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/values.yaml diff --git a/assets/fleet/fleet-103.1.13+up0.9.15.tgz b/assets/fleet/fleet-103.1.13+up0.9.15.tgz new file mode 100644 index 0000000000000000000000000000000000000000..889602d22a00662ddcdd2c73a51e42c7518a6172 GIT binary patch literal 5344 zcmV<66d&s!iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKBxbK5wQ`25XJ(Lecjm6@DHJ^Y$w&CZ?IcCy)Z5})nN-qzNZ z2O?V%Vi4c}pd5|Y_t~%TphQxX<=BpLvnxzh#>Au102hQ zst@irk_hKQ6HNtIQ2-DVO_&&kG0~b~NJWR_TC^!wnlKiDC6{(Ali#OA$K%EpF-lE` z6q5)f;qe${V>SGgp`wJLi2}(r;kKo0RzlUDf-wC_z zF8pBr{GIvpMPpM=QdAj-6B2)y7^88a{X@)YTTD51jwr?m;#EojM zEyaW;sM@26j38#Y(kNAEaVS-r^a#twq}K?;&vJDF`r-F}{QdBr;Y(yl6t>MY_~i{q zMt>?mP5J@Jve{=h{9V}F3lhwbC&*)pDhk>V3!YGIYmBHCnF^7U0mCE!aF9n-e-wiN zKw>5o)k01q7*d8&rxSK!!+%uAS4vRDl4cIxD}xfeC7g^x z%TawP0kDGq@AVh?|K8q)|F0#{l#DQ{?MuIX+;#~7G#N!O@mm7GwmHigI}?n?(+J)S z-wAytQ6bksKt{;*o0V|ew%%t$--1^{7K{=kj;PYo3NX-Pge)I08pDV4QxHR&^XGhk zlA}fihi7j~_vFaf*A3Z#w>KO4O_Ltr@6fN7?uKs8VWM5#Q*1oW7yI;6|f3sXhW5yu1u zQy_4IjKMV*Hyn-*!})NI4vWzoqc-A;qk@|;>M=@-q+KddW0A82v^2;a9l{{z2}1=# zDN-EhHL<+=vQ}v8|*9kYEc^%iE12P)uY9moi779Da6y+7`gi zkctGiVMv(rrCQQhyZq1*-sS1VTajP{t#VU4xI;*Czc{8LAHh&a7>gSaLyg>xQWe8A zgex@=7$cE-fJD0vj>Z^YQ~ri)loPAE^!8E<|1W;VrXq(M!nGYBW8!8;AbBQujYd=Qg(tvk~2%my}hC z9FLKaX#~A~SDEpe?fEzLcgUppH1!)M*m9q4^G2P{54l2Vg#3^H`9nsOx)Cz@G5R5w z?8oQ_v#>uJP_al;0tz!CiN*vNRd&u2krK)k2njR;k|Yw9LgfZ_Lm9I}Fypej-1Ho9 zapo}S^$)_X`LFZ+cCS1C^PnI0_U%Wnzuo(t`TE{{?r(=`5^skE`7;p{76bz!v|07c z@KLW&MP^4~0I&s@V}zOZT0(7rV3hXQED73{>!vFX#4 zB;tlEEs>-%b>70H+=5kOp+heh{U8*^61D#rSRolfN=(xLAR~IL9Dfc*ObiI~2Q|&C zfCbNNkKF`%Rx$M)eGA@+1U*-=%ae)3$PGH;h6k7$=Qx}Tz3Z72G1_w?V%0HQsXfYR zf*mdrY%64jgA3jXwzdM3bzyUajNtapKs@#P$drL5)bJ-W^MV-xf$*)wUvet3BDCzO zZU4K#*Rn@a_}ky@AkHKelIrOZBT5+}@Ix@NPx!Sp!wG||Ueis4JC-bNq#(gTJ}S7W z{W!%5vIx3CCPhX@MB}l^zCdlHmbw&&R!$TEwzlBCJ*yA~M`s@(%cK}q=_7I< zS;ia-W~K`hQ%h5xi(Ij(B_c&8m>4c9r(ll9A~Em}`P8TnG=Yg=d5SgaiwQTo%O53^ zZA;-^d4T~M0hAl=H5N*r5j_sJ;w(QiXhy*n_NMhu_IU|@8T0F)iXHF35u~1ls z7`*Iw{EA7og>5TGOTgWj+Hj3VaF<;==cDvfcU1h~d`B zFF9eY`ph(RI8^2T+|*fu^tEYrlb$ct`M=XFQj$HRF>nR{@AnS6HU7WX?;dRU z|2k5|*53=RDNiDBs^43Z1u1Hh5KW?)A>ZEs39hA4V`WejJIaaRANEXg8#qO<)yUaf z5wvb^p&+|^L{A8JcdhLL*xa=Ec@S=I3$ZhY5Y8==cXv@^4NXSPz0b{)9*>5`H9KZ? z93?}dgD?Y7DrszSbm$rk_IHuTu+Z-4Z~^6-#j4g}LyZLp-l*2l(wCH5tgu|QBACNi zQTOg{26c7{H|>2UK+B$vt9Z?7hPJST8bHe|`m*Y#tXkWh%$L=7rPUVh_}+@3xrr;T zHksK}mi05MP;FN=h%9SdtyO|BpV}K7jD@&9#~5i7O)i#~mwQC@I3IjCKRuH;q@Suy zYlU}rE!*sFbFlkAlIrrmWJ-B91z^?v-@(paUH~G}%I#OBwlPpu6nG!I+CwPb! zpfZdjYs>VOK)cWFBN2aL;rRv0vWDg`ljOc(sYjDo1;%nH00kl@nIb|t%2`WuhMCO4 zwEb}29lkxeI6FK#nU(lM*DEQ~sFVpXq{xzU99DCEzO{|bke@T|g$`exo}64>ot?iw zJA8e3d2)4p@`n$vuP#n5FWg3&jeK`4G5I#NpYa@0K1Yevdm*eN|cN#GIR&HLjA z5M}{1bOz?nn+R)(Z$jtj@bdE1(BDs2Z{Hs;Y}nE>v87$PHxKaQbIuPl#Axm80QT@u zE7ru|{$keS5~f)}sQl!iwE(fo1X4Y*I;faa~bGIk{dTCx$B1ztJHa!oWKUaLN-jG(i-LAY7ADENg{P6fEF<5?TbWqw&*Mquyf*2{sV@KJ zfW*)32eLx`@9g#Z_4hx$jsM4bQdx%EwI{hA3rT-<4;Qb$SI+&cxEc0JfnfL$tFX+O za}XmLwF1a8rMCA5o4cXe_pnOjGDctH#mg4~zy#%>l(7J(<|g-8F!teXt>w&itL&$j z-o#07^+=%Z#?V2Wy|j1;u%%10*hisO3Fc8RWeg}!C?C1}d5v084VeX9@AK&74-W8y z$VD*pruAa{_w*J3EE47N3yOU35o2ws(KWlMC}hi9RvyLQ-iGJSWil*Xsy=Ftp1O&e z2<)?nDi%T}l;2NIcC61t73(2~FGUA+6GfwS^J#nK0kcxFl`bO;$Mhb=3D>l<%SeB@ zJznbN(W_iBMX%oXzj=MNuK!i#-1BIHEA+qaPOtv{d%wT8dH=VT)cF3dwpSj&svpH0 zAKzbTm;UB=;MYxV|DT^69=|;a)8vcvvC95?uv35k+uPsW*niiOwtRq%KO#%Ph5-r! zmqS%4)`p4+!w_aA3C_Lmoc}hYIny*_I1gfCedGQzCborb`X52?vJmBfd_*}$bD(5y zdMrDI6P}bj{Cu+t24Xlw3AsY4AQjfb&TTaha$Glb6jW;oJ|Xu&iZE^O6}xq!ywl{H4vr*jOYZF$B4j~j>eGSNTLk@P+(^e znNUUNkUTbkpGd6P6mIm4n-gXJ(h?eoa2bdgrQPJfm1`bt%K}L zDYcG}$&{0n#)Ppc=v;E^gserib2CdegW3sAyeoP^2Q&B28hl!vC^$3=AyMT_Z~KKY zbA$Y&X}V;Iwk@GhrIE)|hYHCPs*A;UBc!rM^KVS!B)2}YL13MW?PxrvelwW0B;}c( z;rhl?QBhrq)M91bi%X$*%v7j+;Dg?no)8id=QgO4T^rLgaxDp?zuMWglk&Gj#BhTH zMK%8Y+rsnL8B%t8fGSN2z2I%4l+~CYz-v&51IzM)9&4&A= zCU?m2liyO|cTo`h{PWMj|92R>&~KHv4}a$u0S^KwYz_ee)Py{oZ53ko@Ze?Wx?Y2LQ|bsViYltU=cL zewSUp?W`{PbOSVagwK7#=Pu!9vYs3`r4k?IYg6kcJ$Gv2|IZx_WTpMLU$_4p?Cxyr zziUbVZ1F$7_I&=<(*66t^mO0}>fkESbWd}|9bl4{rmrUqJgZ~ z|NVpA1^usgu+jh4k{%iVqu7X_74^fb6s^a3R+U<>cC0FG`X+}a!!{n{yi|6YH;UEL zFaW;X#Q|uQe0mmmQj@>V{E2)etKIgT)uLW0TJtDYx#BlD<2U-9Lp2@?tcb}1%j2@Z zJ+WEf!SPvE2O)t~fk@zCu|(={76Xzz0#HL>l4d9k!AX{^&+4UqNNf&R8lR(~Ta#gb zuwaQZxT{sOpU;~Xj^fQ;YpZ*!d&awJb-VtP`dgnj`YQf^zx;QP|DSpjxJv)u@6~_* z=b*p8iT|~h^gKQ*4>SBU_?s*@NI244W_Eaf?~`H(&){?Ngj34e*Z=+u%y+0GOh!<4o0AsDQ_V-0B0aGD@`KJ_G#DYa|d309DaUw^k z&wNE&wR)VWX#Zb(zW&+LgZBRuy@6Kk|L(zF-Tv3@?``(~I@06(alV$c+N9@BP5VC_ zBTHRm?q}&^)&Aexsr&!!8I@qO|JRVV;EZUE(*3@PTXgqBjf0#rqboy3;%hQO`@?+J zmqF#375t#a$QX>67&x0B<)dwo$cUyBw7(5i$|pPtwt(Zvc_w_D*@)nnxWo7V`!0m{ z&b`LAG$=ua_UBnJ48r4!s|zjc4`CjOG!-0vI=X;_N)?17syp^yhkg(aew7{jZ;>$` zb<97-r<(B29MFKo*Lh}tj93LPLv@n{FT(-34qk>jHGd09N5RYg3AW%TBB{s~yg5Em yL6}Kl6#O8h36hShF2%= 1.23.0-0 < 1.29.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: clusters.fleet.cattle.io/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.9.0-0' + catalog.cattle.io/release-name: fleet +apiVersion: v2 +appVersion: 0.9.15 +dependencies: +- condition: gitops.enabled + name: gitjob + repository: file://./charts/gitjob +description: Fleet Manager - GitOps at Scale +icon: https://charts.rancher.io/assets/logos/fleet.svg +name: fleet +version: 103.1.13+up0.9.15 diff --git a/charts/fleet/103.1.13+up0.9.15/README.md b/charts/fleet/103.1.13+up0.9.15/README.md new file mode 100644 index 0000000000..2f2a4c302a --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/README.md @@ -0,0 +1,30 @@ +# Fleet Helm Chart + +Fleet is GitOps at scale. Fleet is designed to manage multiple clusters. + +## What is Fleet? + +* Cluster engine: Fleet is a container management and deployment engine designed to offer users more control on the local cluster and constant monitoring through GitOps. Fleet focuses not only on the ability to scale, but it also gives users a high degree of control and visibility to monitor exactly what is installed on the cluster. + +* Deployment management: Fleet can manage deployments from git of raw Kubernetes YAML, Helm charts, Kustomize, or any combination of the three. Regardless of the source, all resources are dynamically turned into Helm charts, and Helm is used as the engine to deploy all resources in the cluster. As a result, users can enjoy a high degree of control, consistency, and auditability of their clusters. + +## Introduction + +This chart deploys Fleet on a Kubernetes cluster. It also deploys some of its dependencies as subcharts. + +The documentation is centralized in the [doc website](https://fleet.rancher.io/). + +## Prerequisites + +Get helm if you don't have it. Helm 3 is just a CLI. + + +## Install Fleet + +Install the Fleet Helm charts (there are two because we separate out CRDs for ultimate flexibility.): + +``` +$ helm repo add fleet https://rancher.github.io/fleet-helm-charts/ +$ helm -n cattle-fleet-system install --create-namespace --wait fleet-crd fleet/fleet-crd +$ helm -n cattle-fleet-system install --create-namespace --wait fleet fleet/fleet +``` \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/.helmignore b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/.helmignore new file mode 100644 index 0000000000..691fa13d6a --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/Chart.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/Chart.yaml new file mode 100644 index 0000000000..5332562fbf --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: 0.9.21 +description: Controller that run jobs based on git events +name: gitjob +version: 0.9.21 diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/_helpers.tpl b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/_helpers.tpl new file mode 100644 index 0000000000..f652b5643d --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/_helpers.tpl @@ -0,0 +1,7 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrole.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrole.yaml new file mode 100644 index 0000000000..bcad90164f --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrole.yaml @@ -0,0 +1,38 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gitjob +rules: + - apiGroups: + - "batch" + resources: + - 'jobs' + verbs: + - '*' + - apiGroups: + - "" + resources: + - 'pods' + verbs: + - 'list' + - 'get' + - 'watch' + - apiGroups: + - "" + resources: + - 'secrets' + verbs: + - '*' + - apiGroups: + - "" + resources: + - 'configmaps' + verbs: + - '*' + - apiGroups: + - "gitjob.cattle.io" + resources: + - "gitjobs" + - "gitjobs/status" + verbs: + - "*" \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrolebinding.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..0bf07c4ef8 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gitjob-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gitjob +subjects: + - kind: ServiceAccount + name: gitjob + namespace: {{ .Release.Namespace }} \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/deployment.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/deployment.yaml new file mode 100644 index 0000000000..7771db512c --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gitjob +spec: + selector: + matchLabels: + app: "gitjob" + template: + metadata: + labels: + app: "gitjob" + spec: + serviceAccountName: gitjob + containers: + - image: "{{ template "system_default_registry" . }}{{ .Values.gitjob.repository }}:{{ .Values.gitjob.tag }}" + name: gitjob + args: + - gitjob + - --gitjob-image + - "{{ template "system_default_registry" . }}{{ .Values.gitjob.repository }}:{{ .Values.gitjob.tag }}" + {{- if .Values.debug }} + - --debug + {{- end }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy }} + - name: HTTPS_PROXY + value: {{ .Values.proxy }} + - name: NO_PROXY + value: {{ .Values.noProxy }} + {{- end }} + {{- if .Values.debug }} + - name: CATTLE_DEV_MODE + value: "true" + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: "{{.Values.priorityClassName}}" + {{- end }} diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/leases.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/leases.yaml new file mode 100644 index 0000000000..51f9339509 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/leases.yaml @@ -0,0 +1,23 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gitjob +rules: + - apiGroups: + - "coordination.k8s.io" + resources: + - "leases" + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gitjob +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gitjob +subjects: + - kind: ServiceAccount + name: gitjob diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/service.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/service.yaml new file mode 100644 index 0000000000..bf57c1b55c --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: gitjob +spec: + ports: + - name: http-80 + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: "gitjob" \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/serviceaccount.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/serviceaccount.yaml new file mode 100644 index 0000000000..5f8aecb045 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitjob diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/values.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/values.yaml new file mode 100644 index 0000000000..f7c79214de --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/values.yaml @@ -0,0 +1,27 @@ +gitjob: + repository: rancher/gitjob + tag: v0.9.21 + +global: + cattle: + systemDefaultRegistry: "" + +# http[s] proxy server +# proxy: http://@:: + +# comma separated list of domains or ip addresses that will not use the proxy +noProxy: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local + +nodeSelector: + kubernetes.io/os: linux + +tolerations: + - key: cattle.io/os + operator: "Equal" + value: "linux" + effect: NoSchedule + +# PriorityClassName assigned to deployment. +priorityClassName: "" + +debug: false diff --git a/charts/fleet/103.1.13+up0.9.15/templates/_helpers.tpl b/charts/fleet/103.1.13+up0.9.15/templates/_helpers.tpl new file mode 100644 index 0000000000..6cd96c3ace --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/templates/_helpers.tpl @@ -0,0 +1,22 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +kubernetes.io/os: linux +{{- end -}} \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/templates/configmap.yaml b/charts/fleet/103.1.13+up0.9.15/templates/configmap.yaml new file mode 100644 index 0000000000..3fd0b15cf8 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/templates/configmap.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: fleet-controller +data: + config: | + { + "systemDefaultRegistry": "{{ template "system_default_registry" . }}", + "agentImage": "{{ template "system_default_registry" . }}{{.Values.agentImage.repository}}:{{.Values.agentImage.tag}}", + "agentImagePullPolicy": "{{ .Values.agentImage.imagePullPolicy }}", + "apiServerURL": "{{.Values.apiServerURL}}", + "apiServerCA": "{{b64enc .Values.apiServerCA}}", + "agentCheckinInterval": "{{.Values.agentCheckinInterval}}", + "agentTLSMode": "{{.Values.agentTLSMode}}", + "ignoreClusterRegistrationLabels": {{.Values.ignoreClusterRegistrationLabels}}, + "bootstrap": { + "paths": "{{.Values.bootstrap.paths}}", + "repo": "{{.Values.bootstrap.repo}}", + "secret": "{{.Values.bootstrap.secret}}", + "branch": "{{.Values.bootstrap.branch}}", + "namespace": "{{.Values.bootstrap.namespace}}", + "agentNamespace": "{{.Values.bootstrap.agentNamespace}}", + }, + "webhookReceiverURL": "{{.Values.webhookReceiverURL}}", + "githubURLPrefix": "{{.Values.githubURLPrefix}}" + } diff --git a/charts/fleet/103.1.13+up0.9.15/templates/deployment.yaml b/charts/fleet/103.1.13+up0.9.15/templates/deployment.yaml new file mode 100644 index 0000000000..164340c444 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/templates/deployment.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fleet-controller +spec: + selector: + matchLabels: + app: fleet-controller + template: + metadata: + labels: + app: fleet-controller + spec: + containers: + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: FLEET_PROPAGATE_DEBUG_SETTINGS_TO_AGENTS + value: {{ quote .Values.propagateDebugSettingsToAgents }} + {{- if .Values.clusterEnqueueDelay }} + - name: FLEET_CLUSTER_ENQUEUE_DELAY + value: {{ .Values.clusterEnqueueDelay }} + {{- end }} + {{- if .Values.proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy }} + - name: HTTPS_PROXY + value: {{ .Values.proxy }} + - name: NO_PROXY + value: {{ .Values.noProxy }} + {{- end }} + {{- if .Values.cpuPprof }} + - name: FLEET_CPU_PPROF_DIR + value: /tmp/pprof/ + {{- end }} + {{- if .Values.cpuPprof }} + - name: FLEET_CPU_PPROF_PERIOD + value: {{ quote .Values.cpuPprof.period }} + {{- end }} + {{- if .Values.debug }} + - name: CATTLE_DEV_MODE + value: "true" + {{- end }} + image: '{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}' + name: fleet-controller + imagePullPolicy: "{{ .Values.image.imagePullPolicy }}" + command: + - fleetcontroller + {{- if not .Values.gitops.enabled }} + - --disable-gitops + {{- end }} + {{- if not .Values.bootstrap.enabled }} + - --disable-bootstrap + {{- end }} + {{- if .Values.debug }} + - --debug + - --debug-level + - {{ quote .Values.debugLevel }} + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + volumeMounts: + - mountPath: /tmp + name: tmp + {{- if .Values.cpuPprof }} + - mountPath: /tmp/pprof + name: pprof + {{- end }} + volumes: + - name: tmp + emptyDir: {} + {{- if .Values.cpuPprof }} + - name: pprof {{ toYaml .Values.cpuPprof.volumeConfiguration | nindent 10 }} + {{- end }} + + serviceAccountName: fleet-controller + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} +{{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: "{{.Values.priorityClassName}}" + {{- end }} + +{{- if not .Values.debug }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 +{{- end }} diff --git a/charts/fleet/103.1.13+up0.9.15/templates/job_cleanup_clusterregistrations.yaml b/charts/fleet/103.1.13+up0.9.15/templates/job_cleanup_clusterregistrations.yaml new file mode 100644 index 0000000000..17d1ba7864 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/templates/job_cleanup_clusterregistrations.yaml @@ -0,0 +1,40 @@ +{{- if .Values.migrations.clusterRegistrationCleanup }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: fleet-cleanup-clusterregistrations + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + labels: + app: fleet-job + spec: + serviceAccountName: fleet-controller + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 1000 + containers: + - name: cleanup + image: "{{ template "system_default_registry" . }}{{.Values.agentImage.repository}}:{{.Values.agentImage.tag}}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + privileged: false + command: + - fleet + args: + - cleanup + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + backoffLimit: 1 +{{- end }} diff --git a/charts/fleet/103.1.13+up0.9.15/templates/rbac.yaml b/charts/fleet/103.1.13+up0.9.15/templates/rbac.yaml new file mode 100644 index 0000000000..361d68c08b --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/templates/rbac.yaml @@ -0,0 +1,114 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-controller +rules: +- apiGroups: + - gitjob.cattle.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - fleet.cattle.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - "" + resources: + - namespaces + - serviceaccounts + verbs: + - '*' +- apiGroups: + - "" + resources: + - secrets + - configmaps + verbs: + - '*' +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + - roles + - rolebindings + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fleet-controller +subjects: +- kind: ServiceAccount + name: fleet-controller + namespace: {{.Release.Namespace}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: fleet-controller +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: fleet-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: fleet-controller +subjects: +- kind: ServiceAccount + name: fleet-controller + +{{- if .Values.bootstrap.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-controller-bootstrap +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-controller-bootstrap +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fleet-controller-bootstrap +subjects: +- kind: ServiceAccount + name: fleet-controller-bootstrap + namespace: {{.Release.Namespace}} +{{- end }} diff --git a/charts/fleet/103.1.13+up0.9.15/templates/serviceaccount.yaml b/charts/fleet/103.1.13+up0.9.15/templates/serviceaccount.yaml new file mode 100644 index 0000000000..ba27c748d7 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fleet-controller + +{{- if .Values.bootstrap.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fleet-controller-bootstrap +{{- end }} diff --git a/charts/fleet/103.1.13+up0.9.15/values.yaml b/charts/fleet/103.1.13+up0.9.15/values.yaml new file mode 100644 index 0000000000..7da936a818 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/values.yaml @@ -0,0 +1,87 @@ +image: + repository: rancher/fleet + tag: v0.9.15 + imagePullPolicy: IfNotPresent + +agentImage: + repository: rancher/fleet-agent + tag: v0.9.15 + imagePullPolicy: IfNotPresent + +# For cluster registration the public URL of the Kubernetes API server must be set here +# Example: https://example.com:6443 +apiServerURL: "" + +# For cluster registration the pem encoded value of the CA of the Kubernetes API server must be set here +# If left empty it is assumed this Kubernetes API TLS is signed by a well known CA. +apiServerCA: "" + +# Determines whether the agent should trust CA bundles from the operating system's trust store when connecting to a +# management cluster. True in `system-store` mode, false in `strict` mode. +agentTLSMode: "system-store" + +# A duration string for how often agents should report a heartbeat +agentCheckinInterval: "15m" + +# Whether you want to allow cluster upon registration to specify their labels. +ignoreClusterRegistrationLabels: false + +# Counts from gitrepo are out of sync with bundleDeployment state. +# Just retry in a number of seconds as there is no great way to trigger an event that doesn't cause a loop. +# If not set default is 15 seconds. +# clusterEnqueueDelay: 120s + +# http[s] proxy server +# proxy: http://@:: + +# comma separated list of domains or ip addresses that will not use the proxy +noProxy: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local + +bootstrap: + enabled: true + # The namespace that will be autocreated and the local cluster will be registered in + namespace: fleet-local + # The namespace where the fleet agent for the local cluster will be ran, if empty + # this will default to cattle-fleet-system + agentNamespace: "" + # A repo to add at install time that will deploy to the local cluster. This allows + # one to fully bootstrap fleet, its configuration and all its downstream clusters + # in one shot. + repo: "" + secret: "" + branch: master + paths: "" + + +global: + cattle: + systemDefaultRegistry: "" + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} +## List of node taints to tolerate (requires Kubernetes >= 1.6) +tolerations: [] + +## PriorityClassName assigned to deployment. +priorityClassName: "" + +gitops: + enabled: true + +debug: false +debugLevel: 0 +propagateDebugSettingsToAgents: true + +## Optional CPU pprof configuration. Profiles are collected continuously and saved every period +## Any valid volume configuration can be provided, the example below uses hostPath +#cpuPprof: +# period: "60s" +# volumeConfiguration: +# hostPath: +# path: /tmp/pprof +# type: DirectoryOrCreate + +migrations: + clusterRegistrationCleanup: true diff --git a/index.yaml b/index.yaml index 26fe8eae8b..6ff8191e49 100755 --- a/index.yaml +++ b/index.yaml @@ -1104,6 +1104,32 @@ entries: urls: - assets/fleet/fleet-104.0.0+up0.10.0.tgz version: 104.0.0+up0.10.0 + - annotations: + catalog.cattle.io/auto-install: fleet-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/experimental: "true" + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.23.0-0 < 1.29.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: clusters.fleet.cattle.io/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.9.0-0' + catalog.cattle.io/release-name: fleet + apiVersion: v2 + appVersion: 0.9.15 + created: "2025-02-25T20:30:52.870304224-03:00" + dependencies: + - condition: gitops.enabled + name: gitjob + repository: file://./charts/gitjob + description: Fleet Manager - GitOps at Scale + digest: ffc5a0eb1db89082a5544cdc8da674861a8afb1a7c769310c7b275062bd6af46 + icon: https://charts.rancher.io/assets/logos/fleet.svg + name: fleet + urls: + - assets/fleet/fleet-103.1.13+up0.9.15.tgz + version: 103.1.13+up0.9.15 - annotations: catalog.cattle.io/auto-install: fleet-crd=match catalog.cattle.io/certified: rancher diff --git a/release.yaml b/release.yaml index 8b13789179..eef6f06031 100644 --- a/release.yaml +++ b/release.yaml @@ -1 +1,2 @@ - +fleet: + - 103.1.13+up0.9.15 From 8119d2af4545250b90c12ea32f6a72ec59489b62 Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 20:32:12 -0300 Subject: [PATCH 3/8] fp: fleet-agent 103.1.13+up0.9.15 --- .../fleet-agent-103.1.13+up0.9.15.tgz | Bin 0 -> 3204 bytes .../fleet-agent/103.1.13+up0.9.15/Chart.yaml | 15 ++++ .../fleet-agent/103.1.13+up0.9.15/README.md | 8 +++ .../103.1.13+up0.9.15/templates/_helpers.tpl | 22 ++++++ .../templates/configmap.yaml | 13 ++++ .../templates/deployment.yaml | 51 +++++++++++++ .../templates/network_policy_allow_all.yaml | 15 ++++ .../patch_default_serviceaccount.yaml | 28 ++++++++ .../103.1.13+up0.9.15/templates/rbac.yaml | 28 ++++++++ .../103.1.13+up0.9.15/templates/secret.yaml | 10 +++ .../templates/serviceaccount.yaml | 4 ++ .../103.1.13+up0.9.15/templates/validate.yaml | 11 +++ .../fleet-agent/103.1.13+up0.9.15/values.yaml | 67 ++++++++++++++++++ index.yaml | 19 +++++ release.yaml | 2 + 15 files changed, 293 insertions(+) create mode 100644 assets/fleet-agent/fleet-agent-103.1.13+up0.9.15.tgz create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/Chart.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/README.md create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/_helpers.tpl create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/configmap.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/deployment.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/network_policy_allow_all.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/patch_default_serviceaccount.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/rbac.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/secret.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/serviceaccount.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/validate.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/values.yaml diff --git a/assets/fleet-agent/fleet-agent-103.1.13+up0.9.15.tgz b/assets/fleet-agent/fleet-agent-103.1.13+up0.9.15.tgz new file mode 100644 index 0000000000000000000000000000000000000000..5beb26afef8ce6f5496c869127da1ff8c1507b1f GIT binary patch literal 3204 zcmV-~414n*iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PI|CbK5wQ&S(CLF1ofd$(huft<19SuFB^)d)afmDcjkrt86L@ zku3>p5a0lyJX($Z`xRa&k(6vD8IL_PtNvh1Km%xijRy~!fW{0_M`VIrcTT57>Ud6a z_Uu8W+wFFTM@RO5x7%(1?{$Yq&w59%j*o|f-qG;*S+_Sh>K;FX?!%g~T}q{ieAfNh zZM8f18xg{}&_q+gRT2QCL=z?^aZ0pi7*o+nQEED-m?a z80F=UTXPGPIn^p!X&v5Do{3wvRoi#GX?#Ea-rL@f?=f8>L!vM;T}Yrd-2zh3zg88_ z`T;45)z@zP$GCSKWT;X}3vJWhn4<8Ga56y&C#DLB;7?TlrBFaLT&09zKvSbT)w)ng zr<0l)tzvJ#F^4FHT6LJ12-UIH#A-GPHmzIrwbvcQy|_2{x3chNJ+;s7<>91`|1-i$ zR1d!Z?7;uQaM*3*|M2M59{(RhXig?*kl!F-=t1z%LZU%&iG@&93proGf2WQT0Gdn^ zn0aIf4&Zu2qj023Qpd?1%+~k5@g9a?b496NUFZ3L!2C9H4Rpyw1o>JAW^ZNYCs3A!}$gP?vP!er=;1uA<0R@OQ{XhKDdFO!-_b zX{ud%99S!_&#&Hz3=;_JMOV+qw*V3+RB371h1-b=~z=#HC&Q7OhdvVDsJ8U^6e)Y-d|dsol~ejcs@ZvksYI;; zH{h4GtcyFiR8>z!s)7S}Z>bAp!VX?KC4Ay$(+HWkHIvi&OvxE41#?k?M8eqI`ZL^4 ziMENRqnV(55>T#Da>5MYxMuFMX2tBBCuWz9MqO<1lxlcHhF8ci)j|T{#yuFi4l5Xo z`jpAAU}A23JFpd5DrGD>IDiY0Avkxir3#Um$4n~iBxq; zg)*9yG{cUuDpJ-gDtO1151?Ue0*j@q_nazi+{EO9dBWQIy6*_WcM^Xnsl*1VSQb5e zVSsqu;RHT@3XEl`X1}kF1ppD;;5@OF#VHeIR!A|U8Ope6a>*&Dni59;iCF*u9tw?R z;n@!*VWCa6KvUbe0N{9xsZQWRT%}XYN`^r&VPfR`*12h7HJz8f#xW_G_LFRGXDvW& z{I{>8Ucu(ORkRoPd+~7)WH>4(iNjR)+lu;Wt%)hFer$!nHlfvyIQeV$pdS{I4Uj>*ba*W?6&poP!~YgqG`CI8ldz zQE`i#ORldXTEMN4H%yRBL27B2VW=J=JRuPZ>oo>1JKnx(kZopU@v~OB>ywAJupY%Q zP*OqEik4Rh6|N!5ByR#-Yh<>Xp*&za8=4@xj-u3RdTO8D%frcb{7(fR(@9Q>2MGr3 z!2j-1Z`j8F!9M=`IC4XImcXff-tS1!3`+Kb95u;^CWggq#t5q%=Pn8S6j?H}uzvv- zi~5S;$>(zEb3>O9CX0m@e^!FGG?&Z6bv*`o)+MW*R+xl~MOCUwTrR_^U|8RMh6&WP z>p9D%<9*A2!N}(J`M=&GJp=&knE(CbSMB)U@VLL9|BoW|`A&*Lb!NT5VZ!V6Hu-)O zPz9!TLtkzD>}1M`PN(NSu;Qj8DT-!EfXd~3nOZwWSflvX3IM9+jrhyVbz*iEs}4yo@?ASg`&tN zZl*-cY*B0Jiv|2p3XRR$>_)a;<%-|e(iEnpqpeJDV`8Bnosw>;9A|(YGQARaI)qMP!6h$*7f|K*} zX0PgNmO?qBDW0UMD7oGYWBKRTjGruk@|2a?U3@);pMX=IA=mIdY;Mxl;jOq4AG4aE z#s863Xu{C+b3Q<4!Fn zvRNM#$^?4dZnv5KCn?IpXVfftYTU16`})sOo2~BWLNJ=nKif%X{(c)9*v0<`!+vx9 z@Ar<5_VJ%bktm89oZmlr$|v#7_cjEuA_NyM>%H}fEu6qFXOZ<5^VjG2@cv@4nse1_ zfVlMzz)Xy4IWI)kiX>A$k*ItO*rs4R!TIa;yg;A&)|IiJ&)mnS9`Rq7Y@h#yA#W>w z;@2+X)~>I|fL;8*d)%J?{r-ObKaRBL|HwS~^_>5+7;Ty9);+2hbtC|2ogxDWr^xbH zO*>N|Zo&i#p>#yK(uA=?_pK}@l4Q849%aZ-W8@}tn1D@GWtt*pm>t3h$3kLclT(Sd z-4$*$cNF!_!Ix1miN+zcLpcL_Gkf9BF4^-1+|Zc+iUf9ryR={~kx`YrlQ!q|{R(=|3$% zZ0rF}-TTx_!SMd5pg+J9)sv(tB}-<=c?6`Oc5m#N`tz61?Ri5bN||CKafWhKONf9A ze(4oIT%P-^@MYN4H3@Bc^L1VPFUm8@C*Ri7Xyi+qFFt-)_4yt{0)Wk-{+a<+<>()n zYCFcRrsfj7HHO>Tsm1=dRlHhme?=jSy*!|7$A5Qv@S#?KUGbmeLBEav!~SrO|BoT9 z_q`R$-Fs{_Tr5VzBjo9dTHd4X_9a_Rwr*6?)^R1P6TlnlHpA((4t{T_*7I8>>@n-R z3!I)9{R&3Sq}Z_-jD*l;42s_fT;4VRe_q6=DZA!>uiKvguLgVm_c+p;|62*e{jYzY zx(9ptIkJ8J&j_O#(fBY7VCVjCuyOvkH`vF2A4L`mItGrg$@stMZOvi_ zo>B$&+Ue>Jmk;}529(=t80)Jne^5Y_P4m}eZC)K-!kuQkaCRWi+!@y`{n@Nz^H#}I z^=8wOZ5^t;u&$Gty3t`HgDUX1D(t~zca`qY-i*nG(!l5qvN;f@M4b$U^|qhdH}~?( zOCA3&&rV*yJB#z|f!o-H|Gj=Y{?~ss80_)?G34OD-)eO~DDdCNa@e8Rp54)o zP}#+xY5pOMN)7f^D5|YD9TgN(OeD#3qG?JPo5w+Ly?Pp%NK2#U?v9#jk!$I$&Q+uP qF%v1=;z&`A_jK(={M0VJm%Z#|FMIg{`Tqa_0RR6#)M<7AOaK6H6JsF& literal 0 HcmV?d00001 diff --git a/charts/fleet-agent/103.1.13+up0.9.15/Chart.yaml b/charts/fleet-agent/103.1.13+up0.9.15/Chart.yaml new file mode 100644 index 0000000000..5ebcf31364 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.23.0-0 < 1.29.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.9.0-0' + catalog.cattle.io/release-name: fleet-agent +apiVersion: v2 +appVersion: 0.9.15 +description: Fleet Manager Agent - GitOps at Scale +icon: https://charts.rancher.io/assets/logos/fleet.svg +name: fleet-agent +version: 103.1.13+up0.9.15 diff --git a/charts/fleet-agent/103.1.13+up0.9.15/README.md b/charts/fleet-agent/103.1.13+up0.9.15/README.md new file mode 100644 index 0000000000..2c5724dcef --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/README.md @@ -0,0 +1,8 @@ +## Fleet Agent Helm Chart + +Every Fleet-managed downstream cluster will run an agent that communicates back to the Fleet controller. This agent is just another set of Kubernetes controllers running in the downstream cluster. + +Standalone Fleet users use this chart for agent-initiated registration. For more details see [agent-initiated registration](https://fleet.rancher.io/cluster-registration#agent-initiated). +Fleet in Rancher does not use this chart, but creates the agent deployments programmatically. + +The Fleet documentation is centralized in the [doc website](https://fleet.rancher.io/). \ No newline at end of file diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/_helpers.tpl b/charts/fleet-agent/103.1.13+up0.9.15/templates/_helpers.tpl new file mode 100644 index 0000000000..6cd96c3ace --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/_helpers.tpl @@ -0,0 +1,22 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +kubernetes.io/os: linux +{{- end -}} \ No newline at end of file diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/configmap.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/configmap.yaml new file mode 100644 index 0000000000..f3e83a89cc --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/configmap.yaml @@ -0,0 +1,13 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: fleet-agent +data: + config: |- + { + {{ if .Values.labels }} + "labels":{{toJson .Values.labels}}, + {{ end }} + "clientID":"{{.Values.clientID}}", + "agentTLSMode": "{{.Values.agentTLSMode}}" + } diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/deployment.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/deployment.yaml new file mode 100644 index 0000000000..582eed608d --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fleet-agent +spec: + selector: + matchLabels: + app: fleet-agent + template: + metadata: + labels: + app: fleet-agent + spec: + containers: + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: '{{ template "system_default_registry" . }}{{.Values.image.repository}}:{{.Values.image.tag}}' + name: fleet-agent + command: + - fleetagent + {{- if .Values.debug }} + - --debug + - --debug-level + - {{ quote .Values.debugLevel }} + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + serviceAccountName: fleet-agent + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.fleetAgent.nodeSelector }} +{{ toYaml .Values.fleetAgent.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.fleetAgent.tolerations }} +{{ toYaml .Values.fleetAgent.tolerations | indent 8 }} +{{- end }} +{{- if not .Values.debug }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 +{{- end }} diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/network_policy_allow_all.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/network_policy_allow_all.yaml new file mode 100644 index 0000000000..a72109a062 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/network_policy_allow_all.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ .Values.internal.systemNamespace }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/patch_default_serviceaccount.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/patch_default_serviceaccount.yaml new file mode 100644 index 0000000000..aad4eea415 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/patch_default_serviceaccount.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: patch-fleet-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + spec: + serviceAccountName: fleet-agent + restartPolicy: Never + containers: + - name: sa + image: "{{ template "system_default_registry" . }}{{ .Values.global.kubectl.repository }}:{{ .Values.global.kubectl.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", {{ .Values.internal.systemNamespace }}] + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.kubectl.nodeSelector }} +{{ toYaml .Values.kubectl.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.kubectl.tolerations }} +{{ toYaml .Values.kubectl.tolerations | indent 8 }} +{{- end }} + backoffLimit: 1 diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/rbac.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/rbac.yaml new file mode 100644 index 0000000000..1a7e8d8841 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/rbac.yaml @@ -0,0 +1,28 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-agent-system-fleet-agent-role +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +- nonResourceURLs: + - "*" + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-agent-system-fleet-agent-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fleet-agent-system-fleet-agent-role +subjects: +- kind: ServiceAccount + name: fleet-agent + namespace: {{.Release.Namespace}} diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/secret.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/secret.yaml new file mode 100644 index 0000000000..4715882047 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +data: + systemRegistrationNamespace: "{{b64enc .Values.systemRegistrationNamespace}}" + clusterNamespace: "{{b64enc .Values.clusterNamespace}}" + token: "{{b64enc .Values.token}}" + apiServerURL: "{{b64enc .Values.apiServerURL}}" + apiServerCA: "{{b64enc .Values.apiServerCA}}" +kind: Secret +metadata: + name: fleet-agent-bootstrap diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/serviceaccount.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/serviceaccount.yaml new file mode 100644 index 0000000000..73e27f0be9 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fleet-agent diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/validate.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/validate.yaml new file mode 100644 index 0000000000..d53ff1c508 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/validate.yaml @@ -0,0 +1,11 @@ +{{if ne .Release.Namespace .Values.internal.systemNamespace }} +{{ fail (printf "This chart must be installed in the namespace %s as the release name fleet-agent" .Values.internal.systemNamespace) }} +{{end}} + +{{if ne .Release.Name .Values.internal.managedReleaseName }} +{{ fail (printf "This chart must be installed in the namespace %s as the release name fleet-agent" .Values.internal.managedReleaseName) }} +{{end}} + +{{if not .Values.apiServerURL }} +{{ fail "apiServerURL is required to be set, and most likely also apiServerCA" }} +{{end}} diff --git a/charts/fleet-agent/103.1.13+up0.9.15/values.yaml b/charts/fleet-agent/103.1.13+up0.9.15/values.yaml new file mode 100644 index 0000000000..ae03ceba32 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/values.yaml @@ -0,0 +1,67 @@ +image: + os: "windows,linux" + repository: rancher/fleet-agent + tag: v0.9.15 + +# The public URL of the Kubernetes API server running the Fleet Manager must be set here +# Example: https://example.com:6443 +apiServerURL: "" + +# The the pem encoded value of the CA of the Kubernetes API server running the Fleet Manager. +# If left empty it is assumed this Kubernetes API TLS is signed by a well known CA. +apiServerCA: "" + +# Determines whether the agent should trust CA bundles from the operating system's trust store when connecting to a +# management cluster. True in `system-store` mode, false in `strict` mode. +agentTLSMode: "system-store" + +# The cluster registration value +token: "" + +# Labels to add to the cluster upon registration only. They are not added after the fact. +#labels: +# foo: bar + +# The client ID of the cluster to associate with +clientID: "" + +# The namespace of the cluster we are register with +clusterNamespace: "" + +# The namespace containing the clusters registration secrets +systemRegistrationNamespace: cattle-fleet-clusters-system + +# Please do not change the below setting unless you really know what you are doing +internal: + systemNamespace: cattle-fleet-system + managedReleaseName: fleet-agent + +# The nodeSelector and tolerations for the agent deployment +fleetAgent: + ## Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## List of node taints to tolerate (requires Kubernetes >= 1.6) + tolerations: [] +kubectl: + ## Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## List of node taints to tolerate (requires Kubernetes >= 1.6) + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: "Equal" + value: "true" + effect: NoSchedule + +global: + cattle: + systemDefaultRegistry: "" + kubectl: + repository: rancher/kubectl + tag: v1.21.5 + +debug: false +debugLevel: 0 diff --git a/index.yaml b/index.yaml index 6ff8191e49..983e32a3c8 100755 --- a/index.yaml +++ b/index.yaml @@ -1947,6 +1947,25 @@ entries: urls: - assets/fleet-agent/fleet-agent-104.0.0+up0.10.0.tgz version: 104.0.0+up0.10.0 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.23.0-0 < 1.29.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.9.0-0' + catalog.cattle.io/release-name: fleet-agent + apiVersion: v2 + appVersion: 0.9.15 + created: "2025-02-25T20:31:42.450101918-03:00" + description: Fleet Manager Agent - GitOps at Scale + digest: b79e59689ae444b4ae5693db5266bab2437937bfdf60aace8845a653e222a0ec + icon: https://charts.rancher.io/assets/logos/fleet.svg + name: fleet-agent + urls: + - assets/fleet-agent/fleet-agent-103.1.13+up0.9.15.tgz + version: 103.1.13+up0.9.15 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/hidden: "true" diff --git a/release.yaml b/release.yaml index eef6f06031..16aa1e4d2f 100644 --- a/release.yaml +++ b/release.yaml @@ -1,2 +1,4 @@ fleet: - 103.1.13+up0.9.15 +fleet-agent: + - 103.1.13+up0.9.15 From 48dcea1f61b2a28b9e91e134b12ef2ebbe7e0f9d Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 20:33:03 -0300 Subject: [PATCH 4/8] fp: fleet-crd 103.1.13+up0.9.15 --- .../fleet-crd/fleet-crd-103.1.13+up0.9.15.tgz | Bin 0 -> 50477 bytes charts/fleet-crd/103.1.13+up0.9.15/Chart.yaml | 13 + charts/fleet-crd/103.1.13+up0.9.15/README.md | 5 + .../103.1.13+up0.9.15/templates/crds.yaml | 6859 +++++++++++++++ .../templates/gitjobs-crds.yaml | 7690 +++++++++++++++++ .../fleet-crd/103.1.13+up0.9.15/values.yaml | 1 + index.yaml | 17 + release.yaml | 2 + 8 files changed, 14587 insertions(+) create mode 100644 assets/fleet-crd/fleet-crd-103.1.13+up0.9.15.tgz create mode 100644 charts/fleet-crd/103.1.13+up0.9.15/Chart.yaml create mode 100644 charts/fleet-crd/103.1.13+up0.9.15/README.md create mode 100644 charts/fleet-crd/103.1.13+up0.9.15/templates/crds.yaml create mode 100644 charts/fleet-crd/103.1.13+up0.9.15/templates/gitjobs-crds.yaml create mode 100644 charts/fleet-crd/103.1.13+up0.9.15/values.yaml diff --git a/assets/fleet-crd/fleet-crd-103.1.13+up0.9.15.tgz b/assets/fleet-crd/fleet-crd-103.1.13+up0.9.15.tgz new file mode 100644 index 0000000000000000000000000000000000000000..d0b7ed5eb4f11130986462e526d31d70f0ec16f9 GIT binary patch literal 50477 zcmbTdQauOywr$(!vTfV8jniwLfA43XeR<|ZX3ogSc^f&# ze1jko3JvH#&kq$4jlQH3qoJf6yR17Gn*oa|qmeS3`5$F2_TQ?i>~d;W7Wy_u?f^ww zK1pLMYoN<6YuC+|W(FUf59Qs8D7G@Yl0OT+ukl{F)&34{b` zFG${@xmwSMuRiGRLL3lhM*LNhTcWy1KrcjHJ%O)hI&y(Yxa9Ad74@<=h?6zn%fEa+ zpRaeex%|OD{63#;A3GyGU+0lOZugIs3_YJ$7{7DApDTadJ`W@RZCv^0jeOus>e3YV z?+f7Xd}wZsc@x9445OCeo@qmXf(dk5`hje<}dVln$>rB6|CD8+Zapws8(M>X7g-_(L5bE!C}@;N(5XpG+& z-7Ewcfypp047i0h3QJVFwa9$jL!e@&5Sp)Cmz%=DxSyxl8G8xfo3@7nI1W6a4+ z-bO$<=|~SfaERV3ksb!nQWXkLO0drTu1%&sY+P6nC7|bOMOz%`Q$iv8&CM!II{6t(WFQdm z`OKVt^L<}hs4aQ;L9%AgY_<{9Qn=SaF8Lul;n0=wR@>rtcM{Qfx4LcjOhOG-uBuBR znnZ7*)X0cWLc`iQ45H%VNY!#C5=a1-dVHYio9N$mkIt2oAWSAS;{Sm>eJD6b^`UN1 z+O0YXysa#+7eGmT}ltLcsvVbc&h}&Qx0rh!Zu=d@Ed@5vYL{S-qC~f^RL8v=35DNLE7j zXB0`23f2%t(rwX^v`cAT1|iz~7I&p~bZU9xAceX#Eg@C140v?tD_z8<`EaUKo?a*D zlgyQDFOp(A<*8T>qdxm)tjqImoMQg#HCRHDgM4Q6&!B$Try;bo|W&pQ*Z*M7j zY^0wkRJao`CAIMS7rE487bt7gLMfClumxE6BI$l2N&!rj&s?fC660(%$+Nal^2ZWy z(F=dd_$i*m7Y_V3G*vcm{6&2mmKaRVo<|%yg#u3F^Agzj^_v5(FmxP*kc?$?Xt-!9 zHlyQjc^R-&G8cka$W99YTB8d|n+cbI`6akWBRO;_0M#fJ{V80@$fV8qf-WF)x9#Cb zhd+1|eNVglhDzpXyZ70d<*!HbV=gP+4sVZ-=2d@n_`rCEm?bXP%)9x#QVVH+XFD2i z=nPWIuPkO`SJovSwG`Oud19e&9hCCJ=N0#v)M=qeNO6^-_q<15$FW(iX(^W&GWXY` z#Y%EhkMMnrCMspC9zE5(r$kiB4C~X;ixj(juJ_0 zZB|D{ik4D|pzF4_^q0^6aoT~%C|(iiq>apJ0pu;Hg<9KezwN)g95d}i8_=_4ULCeh zWyBu=4^g|;IwczkkwK#HyP995e-ddW0!-IHC^(L5(x>uJ2ls@WBpTu9-beK_{!U-D zMU%tUQr6GODk3@B6J{AlTLv?GA42dnhZJm`*@;Nr(Uq(blmoP^3MlWrKk!aPh{;%u zUs33omSB&=nx`{(RDQ`8$mI46JmD1VG1>m5L{Ildw=uvc4bd0 zFQanOWqWZKFP>iv)c^qQo+EZ+iUTl@(7;xYbp@|^JK9foRGlBJ>ins2bDf9RK5LSH z@0<8V1y$V`$)-2;uS55rhFHl9md=5g2!W{*av3&zKiDRMA4;x{aTtMisZ-T?I=gCa zo@csQ;VoWb+OVgD(>Pulr_{<7T zg-xn&8jo{k`osHfe<=WZ0PV#G1!;b=4z<@_zx?|BOS>AjV+_MSxV26h1mABL{+gRdcbSMQ z(t5D1cjhqxJG!5=&2H}Iz9WQO{V)LZr~`2K;G&kW%)88w-i9TFakPDNq>IENG_%a( zg&&+lBu&8Jx;Ie@TC7E{Tw0Ikp+IG%nd|a#x8kPepidS3Q!_iSP80C3{u)Jc0EFR8 zHOyM^`!XeT@~3>;C2S9R=b-q}+z2`Gcl0Z~u~RXa&efmlP)q0;vd$Bdgpo=bqS52# z#ZP$B={iHl&EC37+t`lPj=9ve`NlnDYa$avs#oRmUGXlP-j7idbVQB|s`YQijDmPg zNYFY`4MiXPpv^6Pc+%q)lAytZC#5t4u?&^&b6iiZu`E^ZEyF{iA!TLmn!;I`tQF05 zO~_fN7twg?3324cP*T$?v>FsoZeI%?QVu*-L^E`jlu&LNf-%t4<;MFIIbY+rZg7Gw zYA3a;qPs+})J!U=VxSs3oYYxo+#CXxd`3b9*7ak=^Bw6sJa9!^4bdo|Bc!Ot!iZ07 z>9w+Y2`A39WVcqVexqdf{Ec&bB>5H9u=E3hy31I)2#Ic*u+h2~XXAEe@T9Qxbq%qd zM4xK!STIrCsMMZZLNk2Dm-2unOhco4wOchAZ#S&uJ++rrmsA8rLn>Wj#y_g^>>@zQ z%TPyH+x&Q%+QqY?_rONtMXZ(0f&>XZ_ExQn5NNnymrG`uu-O~VW+Yohv%Vk|#LmI# zz+yvJ^Bm1v)P1dgYdC2+fN<3tg@bQ1L2^-(6~%dqwPh25(-0g@MB==GoWUYmXJ;27 zVRlbv@p<;Xlz;wJ%{Kzi9+-jsbMkS)O0 zGy2Vw<|2XPifk2{ef<3AhQxFHJQ_TrBF8w3Vn5#mx}+n!ZJlxXGpG>^7gsLdZt!9sQDR z?7mFsE8ggDBe&{Z*%Oewg1Bv1@?AZq4v05i4xtdt_G2BvpWuOr6DJx;8@mS6Th4^R zVYC^O640i}TbQkCXqHE)wi_7Y6s^PEdc!hWRgls>K8yx`VDG|G2e`NDyeJXegn!qN z*@zicQ7You4h{w+jMJsJVpryXsxp8<;4gJs(JuQ>dL+!!>#bLsNdapkoK;8Wm<7KP zgF}0GyQ(DVt+k;gXVy3R3bpoUI1FfcJsR8(*8nE03!O=1?)qQws6n>RSP}wfQr|(p za7=GHdioaS-7kBAiaxh9j|( zg&n(>KWbT2mj3XAgPMTcoq9YNv=9fRZl~^(Cmp;PTCXuwsws#QS**KB?<{mqBGWv8 zBmYT2KTHY8T;O2|x0R~kN@khAxGHsZa` z*vjSq`7-@+z5e=o`1w5Yc5g{RD{zf1QOz~pi?!Y9?rxE1{*p9%S)C}a_ibuF=RyAH zDm#Rylg+>gN^=(*p(ECM>AXzKyKbB1nJSufE4SNl^9-g+tPqOjyK@Y-9j)XM57JVu z4cN%o+c?&w9N`en7+PAe+(`dAk(V00!L;xb+VmA?rRW5#knYh&_gL?==s0?+yOuE2 z@W`S=ZDlZdmI5>_uV;D&6a%c6zXHQ8sp^`DQM5=-Q35<+>B*KjYWAlkFby*Pl|dsB znOhb`+PR38f^aVV@71mNQ1&vd?%b^;SYk{?(P^!Ot14EFmd0Ya#wVYN$w$qMXu?m# zC!VXiTe43nd$37bHzUQk39;1`aS2TDYB*288qGTlf7w$3@I6cpD4sD(2y9gvw5d${ zxcwdlebIV?C{nYU=@wH_;f5O+4=pNwZ2B(+Nz=IeJPsDoYUG3$@_dJwX>h4>i{jDh zi6P#-w=-s%9g4H^@At{z&TM+-ZcgtkTIlchN(KzwlHa-XP~XRy40i!uPeZmjK3|8K zrMjQTcWSWB))40rm?H|Qs>?g#I?wqoD7X~AuA#g51J z`-6ka0`Ug9T+d+NxY8vodX5ggmh~2Lp{+UOi4MbeKN1;cCiY1tjGibM&fs&SU5n_I zPEP6^qqDag%tL1kZK|pNdc}3O($Y(2HUKH^F(S%Odz=*RM17Of2+~qgc$Fbv7^OjV z^;5Sdv_eIRR&jU@M&8zPpC8$pL8zXbnQGkA89KJz?frXH6sOXql^ox(S_sL(>B*FL zWat++ARpO2_&T|zMa{LCqM6V|r<)}xilLIFQUL6}tI6LbRKx2opo$ z)B7R%T4!a8O{+C$OYuOw7vDcE;<@`pehe0EPs`L^|8hqyrv>xpVK+#LTB{{FdxSBw z!@Qptn4Az(`5HqAe*jVGTrTw5>>EJnHy^M1!Wxr6HlsxBCy8DR>b1kwXI0dF1~jiq zHw4nsItM1=+1(d-LP^^5Q_13Bk^P6H>OrX z!`q6Xxnxju8rmL5FP3T8kp`z$wiei(MsptyW)$(bq-NI5UGPWPf4rXW#j-P{Q_GUx zYtUa2`hHu;k@^pXfaB<>bU`|35e|iF;Pj9Y58$}J|K3{b$?7Ql>y67r=LfRi??(vt ztdYr#;JeasH=_Gmdx>M+MwH*7t(zI}v*0d8XRkxc3^E1I=dB-Cj(h%kkE{CCtSvU{)XZ*FKM!avcr%JAvlRZFsfR(n8vL6<+ao}`&N@IlO%KB*bLIcaXu;3b z!{Gk6dusmg`0WGgrEsfU9jrgr?>g8Vs{i++J>Zx9K5K_TPK-o&vXYRf^L;^1vsV8m zJ>UPX|AiN=rF(MbYXx#1FQCuzEEW_mn$F7P{#UMti7J2kds0y^qJm&-^hys zbJ+iLQkYwi0Pl&SDDdA&Q%Tf$e{9n&&LnEtr$Aw8`+?>WLmfk}BEpi*2Y}57muCf5Wd2E#wC4 znDl+`gL9sE1XD1X4q7ci7}_;NjIuMI@lw!Lhd^ZsDU|kAE=PaVkJF~W`ltpG)>Dy< zsYsa{(51us0TZgQI`5NMCf`da0==@}ee;Bf0oNYr!G(hsAyg>#4Jc zjY;p6PS={Ud=_I0#0hA)$F490ANqg5FW*S*?C6nX&3}AgUbYSRp2p#3+aJ3L?udTM zOmKRN*tP!^AZfRFY7*cdY^&&M4Qn8!QT7{)GtN06lFf^a*5qB3`q98w>X;wMjfj*d z7PhN}rDjLs7tR%`*SIGLH{?uuy+BWO`82V^Z$3Zh>QwN85Lt!RQJUehmeIL^gCh>H zf$#bAK$~1<^ms$2N0MRIi8|5p+35XpeJ?zX{0=H*M+M#ffPJep9TSW0;rz5e14IX5 zn~TY&ix@$ar3e=SBVu~GCIG^d2!>IC_FV)vj;2peZYI|tEL;JTP+S)@Tj;MF4Zf0> zo{xSg0RMz8}MIZVJXJ9$DikVk*3!N_F*+yL~5` zAplO#M3o4$t7o5dk2qKoT*vV9FRNha(@HC}L2x|%X+>u0!<+4tm84{u35uY^+v*|w z1vT#jY2EMsDG9<&+yuliLQ-*~m#;M^P?4KA7ShP#>ey3~8t}5^opCY5%7$N3k2Ah} z8B->)*dvkzRR0`9h_#i6%-Uk9YIm@YxqYVgM8tB@VW`WPu5Sn{Dx60ZI?ouQM8I_F zuJP|WC9HJiZ^>U^y>zMpsY5J_Uu)O6EG{fi17Q_$eyJsMWHZAqIS zSjDZVn;R?w^6hE|1TAYOudneZR-1L8iRd;Px~*-(XWuRR)uO%&-?N-Ja#LK30#L~@ zr=xn?tZ7H8PN=8uAfO8*0;HE;gJgi$Jod;iz!X~!tDQJp?ND+~$Os&D=bSU>hMj7SwXDl|-qn?;kaU^m*djnex9{XUNkoZru?5}qDjJak8?lg=;C1Bx7- zhT&uZu6F1drYT%@OIP+;z@XdTyMnq=YTC|Jm&S+uYehV*d0}m0QL3HdriTHaT}1&v zm)<>_Mkxh86+#=HVZIxg#dq*5`{x*RqX^K7^6E|&xZ)f}jvyY<#~|z!i%v{tGXW&m z3Ei1`vK{MxyVg|bxK9z+y*xz_zaZl_`!G)=^#VXTfq*}WIvr#3>S5fPfzTIHOu}Pg zi-|h7^DY0pDl$>0FpY9vK(wJQ2~N3m{V!jIJV1rv9-QJng64Gsl(I=*Hjw?wio2N& z-RupSTA>#bmnbhR7=ydv4>0u{nTVFU@T=tNC=d zoL4K;>WxTM+uP76So*JQg+CCqs_M!}Gz|ry!Mw5);p)h>t*}F@h%cEG>=qOV8?cDw zbF4Mz{OyCGKjM>Yf{oAS7UM{EU59P6Cuh0u-G0%}|M5=_akKpu1mXEWtNkjAK`k+?tU6^ltlJ>HNe1Y zd72~>IpL&O<4VGIH=_f!V=?(>c0NUw4EKBq&mfdJ%}H^bHA!)#J0(hSKu_Du%l)yQ zmm}XYGHXw?VfLs=w7Ivf$n7%Fg{Q1q zo|)ZLaIzV=JVAnWu<_Q84rVP{tAOVTL>jLKzsY*+MqK7n^L_g#zD#R&KDKUR@;VA< zez;qKEj39P=Q6$#ub24Ttd2Jk2FnJdqkQ3m_;JJ#+JL1C{E6Y%Nm@xTwb_wwwSgc- z3q`p6VIrk~(S(m8vjjXAua6}Xht3#7%0!v_qLBHBGY2W%{*VtEbwVXsZmn3mb#z+) z1&T!I=%?J%Fqvqw30*zCQ&Cpvd?Mnxo;f_R_A!>SD!O=HNgNG89i0rFIXnC_@b-(a5L^LN%)QovIk+@(@u~ zH*0skMpH9g4Tj_pheVI(fIDIJAEyhbOM*&+&kJ4rYK81J_XR$mtJc!OF;?26b?gG7 z2JbOcf)03~rYqZ4K^SgyN0;5Ue0+G)`EAng>&lXks-H1sN&=ny$}j4ElRv*LbjE6Q z37z!N?MroIC)?U8J2rjP@f5tgL$+S+)T)vS!2_Rk6(fijNweY|r?f>f?i5PCuQFpO z?MP5X>sm+pGJVvRf9cY*{ev;r!8w%&>%YLD&ZA2exwu{^i z-fay6!hP$W-<)V14xJG77wJQRNLh?j)YuGPaLOAIe%u)yPz^U6Tm(K*s_ zeu=JK?6MuA*_1=JM;UfwIM+Cc&)MN?|sQewWkQx-*U0XgJ`e9jl7c| zY`@x-sBsZNJqR9|EJb0aT5Q zRMSl^5R%8v9WJH?f%Oun8V=s|1$adImWgUjlf|mds0vQk^?>z+(5Bry0te-d>-^iE zt=Dt1&yq#l^lRa%lIA6Y$MJnq+S`IrI&3xz5g(RHT^*eK%dubnu5_f3ZlKB;+kd~} zr>C);f3*cVb$*?l>-~@Mrkd|+ zV0ufLFt{%;WpZr%&v+XLOa}Z_K9hETW#=;auj$sYvIz1n2LJVnD_OlHVmFVK*auPl zY9*FL4ASNK0Ala2ldVfEqza&B@)G`Mxlwmvg;;}{SnqVX>mMBL=Bk|L(> zWGR*N{B3KGAzl!Ba#KsJiPSwM<4<9?JJ>(>hyyu>%2LE6<%ZIOu%o z2<2R*>i%`uybB#~KmH*0LbnoIHF1V0=&Ar#HU>yO(*fU14p;DKw51~M7-cEdL{fm_e4i=NrS`M`T3uiU^`!C#`3t(hJ%l$E(o>u26U~NN1h?|34!49nzqxZI0^9~A* zN;aq8yc6MFdQaHWYm_N1`7fxPR9VmoY}wDr$NMcLRmYKn%91$G%$`h7%{Mgn2NKT2 z-3LZc_2N6$`+ej`T|0;+`&3T;&kF|c$H>7Nhws-V1Ivf+&)N6m+l|0SkK@fv?#~H> z>rwGLd(1jIJ!xm<&iCEb#>xu9eBG`ho9~5pesaY?N@oH`6Qj`k?hP(lM+>>v)Y-Zhy|o zoFx{bN8jX$Zi+bkL*&bg$7M7n;y{uG_ayV)KG{#(B(^T&MU8@n8+yAkKOdMGItP@` zXh9$HnI=l0VnJ^3c_5#Z~7{Fz=F8O0z^Kp_**^L;)slzO&M3LDV#b}Z9L zsOx?A&qWh>J3hncasOO8aQZ8mxUg^H905z6pZND|)(9d;c=T)MaYtM5A>wC$Xn~;{ z>sfyo>j;iYXMf+b!Cl0|I|#IiJ!XjvIS4C|r6t?n6APsGN*z=+BTX+286v~Y&bv<3%rk+VGi@# zTAsoOOnWBs^fSqJpVw#_3e@7;UQju6c%Z-nv%r;TlIbV2c2I1Yy-SYTk)O1J82==R3L`;Nt8x^)(<^l^N6GU^2r*c=#DG;{%fT{y*Qd9NCkWu zRSd)eRTx0q$cIV+*Rem%2_Jf^OqTAU-w4=w1j)!Yf*QrD8uC}l6%(KXmQ`keRz^%k1)(Uvd>gzFSIubdXT|}6~3@d$j}*fgKU8LpQe#6ePIh~5g=>lL|NyF>-t#s$-qDQgdkSaV4a$J6F#5MYSn&jMg8~z&<%ZnclNHUZkrqkU2IYC z+C#+5A2jI%=p5re#q4V_T@YOeC+0msAL_T0_XH1x!f0-7nZeh(Nu9k|K&>#VpC$)2%__%lsW+ITBdqtS)SmX# zx*dlRr4kdZ$;pzJ&Xj|fn>6i>%Wbq;Nj3KuuU_FN*UBnAliDABhrVDVIj-aaW8#sl zba#I;xh4fNPWfes+G!jd%M3ov9Pm~d9*Nqtp@YhS_KNb8Aj-82T+(CsL|}5QK1O%; zvXqwNQG-u)JvQpO#F1EEV?$(Rc~Z17mDdo$2CW29qThlMZ!VL^iv(}89gPME@ynaL z{ML3t1h>2uR@|CA%Y}0t)tl?}$Von*w?_2hsLScB3osBEKr9zA$9{2n=s?V`qZKuj3x(O#mO!GAerE;h6X+D8pLb+K(6Ja2lT_f-)X7AV zQq`HTE*amv#z^Ra@N|m+mG^0f;>W@XGk$qNnhoWgw9h>*-GO8sP%A8s`RyVXHRzDr zu6)eETFP6Nt%jR=Y$(WHopFQjs2u?LM|tWw4Dw`*` zUL}y%Oa4~m8_5^S{~!|1kB1JT#K(fcL&AfJ77r8dAnJ`8_QMMO6^syt8GIdx9Xw1Z zA@;(Ftq;SuIzCh=mp?!xpYNS7pRYKoJg04w8oGw;JzQuTVAX z9bYx;##)(%pLyiyBGDw=Ve=72g8r7bq~~?!U($S#b>7;pq2HzyVKKx|hUkrEv8Th< zFOBgz%bMoYCxwya)K|uS7LZ7PYMDrnIY+KN`*8K&Xy4uMcD=R@DGaM7?~4F?WTKKQ zGz^q9#n{@d3bFj;Y7w^#bkp-5i~9NDX@hX3X|HcLjy~#dea6qeCgjSN#Mo^g@tMVH=rkqlGpDrjH5>~k*xbO;s` z0^FTXLfiJ`Nb;;RI|cDNrb0I1l@jHp)`^A({fBaSpR=J;UdAj!y`N{D>6&28Z?b1+ z@Y5fSIt9cHg#{wd7@@rMRwQ~uOsZ6`8#^+@?0#IPta^x%7R%HwshzZqe`4(2@x7OH zFItRADvLp#hOP2kO?!W-Z!geHGlzcG8gt6I?GV0yNJOb+!|k{#OV#_cC!4@}y(+5^6S|nWCYR5Hex!i`eQYLw``Tq>qiD`=^Qr6g3 ziT`SW!+(P1wQ?2<{JWL}qPpCqxKZ{t;y?;qraCH1u)HDkF$GtmF0%<=jeDRdVIKiH zwgs-Bobct{+{)EUYj`b1!ZZ+7H9F^;WiJ7_7H?*X(g|>|#5nQ9H4Sk&ek)?I0LIjW z6G~^iDuh4F0Q^&KNjjALZ8BqKQsb)$iS<+faP1=62-~_W&BmtV`7iR#10uqL>%9!V z8~5hwo+Z(`+`CCEv;DuMXKLBzw?9yIrgC|@>PYAHi6JY=sa@~$di3xoBq=D=ocrX@@?VmTR4AO#R( zuGZ2AuTPC^dgTdG=P!wnhK+?dR!==}EhCIfve09)4j=ruhwu#G%lfUH8=9T%n`8e9O5YWgV;`@ccz^Ho!O@icb>X7*qMu<6vfa-+{j3TayLP#36 zL1$oTM=cW<7$KmVo(zols;LLe%fY1ALk>kX=l)iZrdCNgfYu>}6ot>VzgV+d#Pmp? zo#Oebr>;++4X=6H3Hjcp8WCi@tzF2dLJ0kkZ#pw8gDtz6_a4zbpU{J)x<=AY$ zSf6=`vGBu#^YwR^m`m1yWmm@~RaGkv<&H`^_ z`uWT}6ux27u9{YQr=R*&txsXkE7w7Vyv}XK+xsM3{}x;6u9|k=wOyC~m}`(m3^rG) zZR<{z!aW)XULZA^qTo^#0CuXfkMipu3^SVl9~hBP>Woo5`Pc4{M+mM=S7Yf3~`+1^g$4+>6cK$k9f z!!%lV@(#|J1~`*Vv;IFAm`@Xyt^cLS>xE!xt(iXKQo+P4d-Srfk8@Sd0V?XgJ2ugT zx$@0@ZtjA<3)Ru-P65k_%_b3MoD_qc z8X~t?XF!;of%!>4wG3N{fiZyA4$0Gw+>ChA>8LW~M;$LVG~)J@x~Fqco5DKC^r>N| zrX5vzH+f*`!e0mL(`qY95F+|evOjl!S{Nx9(ds*HC@~kSae#!% z_fsaNT?St9GOcjm1YnLibJmJncd`*YDGdf$m-*dlf#6psf7!1Av6pl0SZ z?PAv>Ax7!k5S)^y(&$0ed)E?b9JGSM;9s9ekwvezEIqvR8>s!v*Zb1*+$Woog!V6| zmH_gXg2)(Qog6@4j!yKF(n~B&sugiF^1syp|6hdVdosBH|2M)iz;#sG`dj|#e!S5FVq+FU;R(%xJ}%p2tof5T?l591k39~m!cxlGA4O%=8I{fk zSl&83K}+`?hdYXH%wn4_^`2ud%{b-RWlGpSTXX4oJr~d^=0e*P**Q3Ufwl&l+atDa zGBY8l(ReZOUMFLV_9Jz_F{y>s_4NMtOhV2rvtp*lCy!o*Na=#8M)a(^U%;QlyW9K4 za=RKu9hWHhE_EhfedX+&cQ$gJ%R19#`~8Y*fJtNOm;Nm8t6SXyU>=H}^k7QVk? zbyeJx{A|;W)0f{!X?PJ!(Jqy!?aK8dl&T$pKA(#3C$>;j!^1hgJ6@%pznnthbsI1Y z^W?Goxn4 zi&*1&x9VjS{9)wN?e$f^)BbTSAV3&wTc9HaDlLA}E8wIXf6l?HLwS%8z4O}BNd&dC zaIFi~;M{GrwE!hQ8PHfbMnY;few!r-DY`>D9yj`7r-xHtBV%+2Wn~bRfIc0-NWxpos zPXig@2MLlPxM9@3u{fXLOhNv6yF7xWaLzXCR+pyZg7ymCrD!Hk=!Nhjdz43`shj!e zt9+$H#@ZfT@I6QPInuJ8vOWs4?BmJ1?`pVdTH+_P6642u ze#dr)r^Y8xIpl>AzR{we3rRhV8F%JsnBL7d1)KUG1}3b$RY`Jeo*FC!L6{^hS9M^D zA{KEw5VYX?mXzTJHi2&NMmWoc1OKFLb=M{Q*rwy2ZC7hxU)`_p zYIvfPJ2*EI{B=c$bHywNeOJ|U(aFH$-jNKk1M5?Yx@x={`dvP5M|e|;5|a_`hmuj# zShR_D5a=1W7EwKdj=BPSRe8Ba?`oGMGAGEm2u|inRgYZ1)7HER`4~K9T>Pn8iqEj9 zVp`IoKbZ4DTiT>tRtZAQk{Dn3ak|tKqY=FN>1<@Q@8X?iF^{M-p$@NTr7|ouMXob9=yBG-yFLxu<#1 z&#Ml)@84G(9~Q7M748!XlhEJeaiX+T27`?=whutbA~Pe({9ZGO-OtrfVO#r@Gc>->EJmK1~-eeRH*wgG$KW z(jF8y0QVG+EwsJR>Kf)76yK(T+9FJMy5h@OJVwr{jQmPzvfZZ)HggI@+4!{TkrYs< z%p)WwlC|xa=4OLr${CE>G|qH1oIy-{sNe8W%3hY$IUz>a6s;vsI;~g8<=$G&E6hN4cj%+ zep%zlT5U+%s-3$otqWvsDb@2!_k=`MX?F+$J;nXx*W_t#f2s~b#zc*nB!P82$|1Ur zWYgToH8CLpVGL2|B^Gaxm>M5tuCGW=_Tn{AVDreeQJIhw|5YFz6CYd<7!#ET?eI`` zS~%><}T;`x_;*QYwfuPzm2~wj?%J&GyRj{)vp7gZ3``yF_I+ z6JlJL#xp~WT^hUeGe(LGYI;iw0i2mZIJY~W@9)lD9N8S(MK>LCK(j*UIw3J_B2De{vCPMjs4 z4JkVo-&o*gn)bsI1*{l~XUE9|hVNAuF zm#pP}ET3XYN>}I4OlnxL(LRf!0Tgv1vZ|<}<&TV<1MmJiyx?TwSxK*sed*d-StyIfpEnnb$Q3svr za?JKoA3lW&GVonTb^2$P+F~$}YBxHgcVdsNFf(yfj%NJ(2g?*?8ZL;f1}~kvJFXT| ztw?z7M;z7y@#)>hzYoiI70|+sFu&OpI)K3D6h@#yyDQT;v~QUCZ09&bwOfJba&I{A zN#Jl__+Ubm{Olfu>`_b^J&d_icv!-t+6lvQUy)!kcF+1E@mzm~2T<+wI(^q@mazLHl zGVo{Xqtei3x!t{x7=d+o&Z(SaR(!0%>?E3931+^!h~(8olgZoFT^}y^y>VG-q7hJ# zFxL}XaC;=C^1i8MmQ+34;hKu!5Hm|486YDyt34aFicM`gG6K`p?r~>IxenQton|Jl z`OH(YMTZhu5i4o4A0- zQC{ACP4+wsdh2a0qEKM z?qPt{`R^Ef#C15X!R6ZrCBlxMfa7o0zXO1)-GrZdAM2Y(==dSrsqaQe*GrpjH#rSC zc<=JyVc5G~8?2WG6l4r#E^gs+3f^)KS}I)P=x6K{r_oVwS3vj8Na`j2?YX{XVTJs< z6si+pAqVLOTz-HG4saJ93|enEwY@6W+Qq{pR=>){EPkEiFM6(=q5 zZXz(c-F-3veSjoFGZ+Dh?|4#op=10;}~R z^#UR*P-8tvVQPS7hm&OfqQYblL9;^b169NP$I@M11zvp@iJO31lpozq_ zmJEGNATr({=ZBF5B5eSN0Amk=3tS@6;=yoP4KjmS-@PEP<6@YogCm7N{vy!h9PZ(z zm*1}=;<+OVI{1vPpZf(F0~Ldag3HmhnimZ^q8SZ6$9btx5crM26@_?OhND*fL+-U%wwLpw9&X^x-B_5m?VqxThcWImGh{^R)R!gps|ChW8@OuVx^~VF<$h3Z?r9 zq$#8v3uZF{ZVhH`Wnvf<_zzzGAVwye$w4#b2_a1On$0_nOEp79ceB%A2DBv=g*fyy z-m9B$ZZ$o1PYtbP1L(fyUsRnu3>rrKfJ*;C4u@S(oPv`Xi0m5YzcNF?%dvlaX)p{n z(#?TG=LCM9%-A#eEY#tV=qcT6p%NKC{}G8)>_1RV?oJ4PpVM4WM^Es>j`C>o_5JTS zc@7-|EhwF|hhET6{wP%d@#7?c`H70$;N0B&>lsx;b`tqIVj6mEi_^(Ypq~a%7Tphd zS4dy_|8z66z`cqowNSIruJ`Z>msq5?k^RDc)1(x8hoiHHc&f{9P$x;E@CKmoHG^h0 z!w8(A|2pBNc{rZV%>Rkc>KeBAf}5J+o}}Zw?F7%*Q1=m0cp12q8_`;(sfFk<9FMT^ z9sc%ysa@=S{y#~nsFk~ip6TjeJx^N~HPO{4fyZMT6r426ui+c}|Ls_2{69LD+td7_ z=?u}hg-nXUeo;$_qYQyBb_Tq=KxKBV2f_M*_Ht)gZnx&PDmt3Ah1Z6gJ)CBvA7b>+;1 z(9QuyGlD}Jh7V1xae!Lm^|=ho@Rx9R%R6LQ8vTCl>@0$MB=dGmudfeSr7zo{(=31C6T2|^bU5tv!+G&l;)SgO{K|0x&$65 zbOi27%tBfkQ756s)4kMDB%|*gQ^k-EezF0>i9@ULePD+p3>>gCN_uH)VyL7WG{p?GF}xMaZr2tGyid7odV7 zKfMldG3E4+k@#JXUUoQaf+dp_IIhI3Jx9#0xy)XB23gbP@&;_mTm@m>{6fYhpsIE9 z#m&}a{b17aAEB!IcH6{cB2RGfA)ILzI*6@wp^>+<N*J~Q$x+__1zvJrM} zpUk)7dfp$mIvWU3T|dzKkOtm^EnZk8pPLK^q&NuQl)#6q!VkBXOFXP0|p4jlTKrWFTAbMbxM{AmUyxesgwqAtS?yBcF1 z**;u?ZuLu+PJ^fozV<6JAJMGPeTphKsdEQzN|U4BUgDpM>=f=%ey!;~H|1QW$6CQ{ z*=Ogo&0mRps8+jpjbf0YYLh(%m*8lS=+ybJdl`#&{eYqft^5x`Gte`?t$4p_)J)@r zkLgAHs^$kmHw2%S5RYOw=8#?#-8$Z@x`+H9&kY-^?!|{I7XoX)`yR#~BOT~r9j1v4 z@n{yes4R@4FF9|$!T&B2sURI}Z#wmBfR+-l&N=TH6w7R*nc#qpwr~hOL5HIaU^fgO zcp4LI7WL@iNH{=gOO@b7RCzFE3TyFzP9bdw@7zT?ejGc+6KzaOsi09ZJrg4;^jhPxAkS z7mo~i64G|URX#u2^YKC$dy!=9gi&y3z{KwTL>X@OerIflzOI3ZG4~?DHU4{wTOQ8; z4vGt*{&(a*eILB!GRG(o#YjdXL@da!?FhUwFn)(J1)9%Ipu%&n)ZDc}VkE2DUi50% zT(}#Kh9jDK%ng$3cae{3;vCZqaW8_dRt z&ny3L>Yra$Y#cswz^6rz#Pr7PLchllD3IU$?1%zV9m`pscun+cIC@4)l<%JlD6ul# zI?JhduDjgqrtI2HlvrV@EJtJ;x@loELB!wR!YZGEWJ#l|Y2fJ6E2}E3JDk3C1~Q;K z1AFzwmS0c_NqX)fKSyfF!j!6#u51*c^mccgwMJx3rO;@O*MtWr7>%wwuEfb6J9T-ID$!6 zdN$JGPK_3^qHf%_`r+k0P4rW*{iL?Kly5#qA%i3xih&6D)LXu;s?oY>Ji@tNdh-?7 z?HCTdw9>o|$}!2>%J0_GTEI#^ZOgK}?7&c%;Ou@suf5B<0VdOeB8Hh%0DTJXqQelT z&kwD$7B!Fk?>*bHSt>X2nm(8TzS+v6(I|6z@*JAwBWSHcy^_NFZ}6)j73# zhZ0og%o9|Y%vrvJlRJ`Gi)|HdO@A{q;0C)zFWhM_Fc}(C$$v!ng3BLPN!dDni}llc zhUUT3HM{dfyejQ^=0MqM>960pc$#>SEutvZabQY~E1ptA``#DU2SCKDcrrg}Ua#pn$`thG;N-5WrDGEG1RgJ~xCfU25 zt`RO3w30YAsmy*GUm#{Z)jgP48kw~A9tN6G(2Y?(0PI1qw^uiP_$sO;uM8h0x$N<5 zFOD{8c5;~IN8a!F<^Kd8iIjy(jLp7;yO$V&F`L59vSKMbOr zx)`Bf50998;4bz;NCD?6CR&7KD`rxyVs6NDkDCdk^i-SPljH0+eeEGE-)<9% zoN~K)!Oa5R!pGOiRRqLGsunbC&6G&rzDj+*2S2}4Gb!{LAWv&p+M7J;X#0eOzJuvg zxC=HxK^wujgf+8OY5U3D?k$rxF8fPtQY@7cPn3x{wzbi)PDB1Np%Xt8$h2&h53}45 zlngL@B${&4EZK~&DOd2yaegRj!6n|$?{O_3n71)x!jhNHDt0*&XeFRtV$Rpxm*fJE zU~5{7K8fTaV;)R)w*c5o{_V8jaXzH)x>al!S@64s!P{7*#0=_EEbbbKkh>{$A*no% zn!vGPa>7c{*6apKzbv4v=V>k>7;o)Xj=vcLu(!wr1W(gO~+SN2X z^K8=8YO0yhW`zV8UepZRH&ez(oz`;0gL?iV60;g>z-|RL{PcE;ix_(60LS>Cp)N3X_9yuS`8`8JI2^KQv!zz@&mmH{^ ziWR!=g7@*PQL1)i-m)N+1FYq#fvg#}CT!(> ziD51fXGx3Bbr3b!^x5cgh1+0PdHM>4PE8!*rEDStWzSL3WTAl-cM zFO++P$zf>Hwo??iO%eFqS#VjtgdAVqgB;nHBgM~`MBhvEy!Sg6h(i(@Or=PGJJ%VY zS8xkC&KG3mIQ?t63QkV?ftDFn+?pRnLKkp^-qIXt3}y0XF0rtb(7@!8TTE(nHaje+ zT1JTSN|D52aBU6?t9ClF#%#l91NIh*qd?;|WS9{0E-Hd-c_I=DBDXt57&A9(m0Z48 zc_1BeJHG^(aAx~@pv;9!++mN#xIIWXm$tKQ1M`^1G8~VH5U2b}!x|8vC3?RWY9Hop z+(S>KN3JF{q0(;R1}<}Zd)Kjh%$Ye^s81Qm0sZ2O$7?>62n#)TC) zLo2G^SfG!2@hw7oVnXySEuh~_jA(}UXAxTspIEHrN_VpsvWu~Ib&6@n36ReeaC;AJ zrdjl4_~AKZ3D<8cB&n(!Q6Cq5>8vV5u{xm^e(BkEY2WqeP(6exJ3|XG?-}X%4kciN zUroBWwR;xv00?o8ZF(^@>=Uh@u+OV`^tH%CjXv)l9F9)?QI@25rec1-J6;im5Y3V4 zp{7Jk)3CK0NpGykUU?#l&{}a9dGl+ls+C!xf#FInEZS!H@R-y}(i}kk#{I4Q60P0Z z_5E##Ima}{dJN^kaVkMy%2MuD?ObEXvI`aC<=5H<^hA$|`=kaNsiWr~xBk;WH8-5o zuQ0Y)0cpi=%*;>GM9N(=LvQ4l>EtJihmq*qN&19g$brjGQ(=GaKZO$xQ6{9tW5;EJ zzh$ca)HP-pE4pS5jOBCmO6QXtRnLS=EX>H?FH@q9TX^Ms-09}1ZpX2YVfROHOD5=> zF~Z6O6KzqG_sj7FkI?U%|JxL_W`v1t@o%VX%?Nv88~ERmdHkOA{nNB*WitXBF`-Di zMT5IOQkgcj`GCL$gJIL#yQwVN+4y0Q4O)LT#~OAF^m)M)Y~{1<4+cRaMt?|g4!_@7 zu&A$YF`@UmaCN)gZ+~<6Q)qU6-Ft7dCQ$MHnlHEqF&n=TK0A`zWLI#v1li*p84-_NwbJ~~Gt6Y;3o`?Z0Vlqg=~?&KWBK-kkJhf$00U-V%ZJTP;y*1{NkW* zq$_R%?X)iEqF?_G6gT|zGEU)z0nj=&p13F->(kWaZmk}$C>@z4GS?0$9a=5lm5{Z6 zO2+mMK_u6${HNs4X75aDm4gf~znWxG7 zr~hs0WyK0M*!u2^`VRWOhkoE-2mj~#o2(0XX1<2yTylW2%N`|Hz3-_l;5`eU!Z+sc zzWF_OGMfk7`w9*~5|eMYw7{e1+Y*}v!Ub8Sno?{6MynKA!PFr$t1V3f&s zI{!BwX|eCF`#0LOIWzukIJflj#@8Oq#42|=nmVi+qA>+e%2xxGwbW(cK2mrZOoP~ zryHxeQ^ckwR2vSL)B%5pj~xUwm0kb!_ve8rUuNga+QQ1%QO=r-qFyp`&AOaqMpT~a zY4WXtNk%xy;}u)KKaV%LjXVi+A-MqOcFw<-Lt0<(-a2^(tUQStXPS2rF7uCZAYJ)Wg&AcjmT3QxB;U ztLt&(wgKzNz=PQB1hTP&RMfMP;wLFfSBqdjaxt|>tw6RAzl~!JwA>**1(I9FQSGKk za#~uC?mnuv@lkBfB6Wp^8LdKyT%4`6nRu6aI5M%8M-^bW7f~-SZ-7|E_V4#_Y5s8@ z@(BCa-$y-q*p2=+=K=bUTwx8|T-p{Lqu^`k6S~MtX!GfBO!?_iLr(-sg1n|@y7A2$ z0yf*&Jt ze%$ddNvV<5`1-+dh|m{t#>9OqybMS$Ds_PUdaKg8T~e#s^Z$7XrW{U>NJ7LEODB83 zQwkor_EU@?ev59yOH=_$TyMlMTEN-c8q0wA*y-^Vc>_D&m)%167B+?9@uJf$m%MuQUp+nh8OOvUT5A;`jwl*lZ3%;D9y2f66?`>gnxp z4}7u+6th`MDM1{%8iH-=;jkppJ6B9_tT=mK4rZPz>n%?UK6M0j z9*(4mOC1A1bTRR4BSM-^OQ>c)3FonnY1?gDi4v8c&tM!}c#@z9?+pP#028}%r1x_@ z9KGpehm?PCJ7WbSH|o*CPCwK;I0d6hid$hM5uOmWKqN>xN`Kq$^Wcz~9vc6=$$C>;Wd#Mwv0ZE0^+o*K(4D01MP*YAmq>i!gd zGH(b2BvzkX(+hO&U=qK2@o4s}4MkQvQ=wir7)AM4r9LT?zGE1-bV*MZ11H!qy~UVk ztOvXX?V7B&92KmpmU!p;k*vWyC1srHlJbKrH7&L*XvATbt1x}HR-YJAWR}8zjnHjw zq8CdbkfLtJVB9j>;*-Oc=b{%eEJ3Dk$!-b__8kKRzI>wiNyD)Pm|XMPX|rL4Sl&|S zZeH-HbaC{=8!94s4+q0r8SUZGHNuu*LePc~tlC&Ykw$vq&C9mjXdVvN{z%xH5j9ya z@g=X5ffn7_ce47?#!LSk<1hxM0`~CvA8G4OW(s-uoQe#=yUoOx_Wj5;EhdE|_L36C zACj!Y)E%@m<*kf`%zY;_j3_nfsbV{b#28PWtz~%^)x`QN)%g_(ZYhse;f0VgVeS%1 z^?u2{)EvTc4Uo7MuYgWLd9O;szGv@mz{4MSbcKcIiLw;hbw2l9KiEtsHPmQXn8`H; z2f>=>o*xm^JqIyFlx8Xv%cwAO*{wJHH#s&mak15{>Ry`92)-P)=8l4? zCu|C*kEK2J4t6CZsDF|IyZYKyx-LWkD8m|Ir{w6l8ipi$BdVo;A`s@lSNZO>?689b z^21hCPtK2orf|bAx+mW+9y`-uWAwYUWwYX=Yvz3FB8j9sa}x{B*?B+vS7{=0m6^Y2 zIAxGM*f<-{cV9q(^}NrQ@nr?i&~3Zuwn>?}ON79SzBl)k0|!(#^y(u4U>R-~YSz2H zx<9M1HNLUAjqLgMg~0*O!?Ne~_8)9Ud%7SQ8zy&VM63sen<IZhFq9X24LR%8bIo6mm?2 zM@$4zdw54yOZ2H@S2*H#Uvw?6L+CO7^w;qEo$k4pu~zcJgyM88XT?KzTeR1QhQMY1 zgU`BZJy*^bzhk_d=wuMGrEMXbx7cQTh>Sky_k%h7GVNmMP%QO1y%J zM@aYU@nmo4Z3WtY3>q6pJF;%M$XPT;D%_vuX0iCIx_?&G%OHuG*3i|2wUF&@ zWMJ5Z<>Z{KrTi;653Xv4Z|`X_i3;I&rE*I~$}2)(TF1pesKvdRjH_}k!y}6VXY{Q# zSMr|Q(rA6FnMF>5y`(8G*94VPKP0ubMjyYYlhUcTb}RS$FdTSrWC9u!k0p#Rwi791 zF@{tt0!fK4>?L!L(!OqGp0xj{LJaqN*kI;E{F?(D5Sc zwyj0HN2j_CMt%RD=*|oPf9c#pVoQ@I>Cnwswu4vmQVEW|aGG)tCes`%nq&iGyjXI| z;ittHKKq0|i*|42O_qKwC{Y4lj@Xr_O+;TgWp&KAvr^1+3+}k1+dyz2lE^J_fA~9n z=+G?us2PvWTF^SMvxSepgBE}O1V4WGf!`oMY1J-BrCnFAdy(+!j)2Pdn{}KNo7+ow z&O&l07Sz}OG1Jq4f;PJ#-&o|^wHaT>Icsg~(OWH#J46u*#=*S&;PiW?4z{3(_7sAP z+Q^OQ*8zJTwX#bsS_*!}^Wx#si=?eLo?2*t^}$2&6D$wv{;}dp1@%_QfnlU#Wk5HDvxMV%l=$`5b-uBHk@58vdlPIjUU7$gAH1WW7!I`v%;x{g** z4y8ly5xt)=wnlDkIF-vXe^lI)ZBCtLNu5xz>&%qnS*ZP+$=~AEr9mjqa;>^jK7*AE zwVB@I^DYCW^QO;k^6mwTB+#YfrK*hR4q(kRlre?;Msz*K)Hk+qe316Ya%Y>_LK2D@ z53z*&JPb3IyxbP z9XX-sJQC+w2ffoDqa@lHx>{J==>E*86esZF%3;GD{E${-SGPG{Ot-Rukx@d{T*Jjh zfZwkwvPsgiD8vVs$(_5z1`dy;qTfGdIJkIK%%W@|VMVrq>0N$MYKd+~|I9^}5>v1_ z;~{|dG?ib&x&fjqJHeLevu&ccf?nCO3~2Kt+ds2sjc&Mb#f7O`j1Qja!|Zwm5d4B$ z%LFs{gm)Y)7vDi*vp>U|Qq~d~_9Aw@S;JVObH~;v;j!e_Hf65hx+JunkrfRJ0OP%# zBRqbR^EsiPt_Po(6jljP+yllAL}ALA zhk@3$j;zCGbBk0ngmw6)6teR%?L@AGCIa*77s2dhP#3YF#y_$9xwp#fz2k%t(g!Hg z(+66chOBZpN6v>j&byB}#w(FPQ0>z5Y`0RB-o#|)42%#><``seWjm;=+49p^JpO)u zx3WtP$uJt%wp(|=TG7m%7>zpba6)dn!c}^H>XKT>M%m=niH?R;b2Kbp6r|$&KOx=0(ClPx2792CZ*V$rO9`*RJEMZP4 z-HfK)i4RQ2XoL7wiI%g{^v#_i_u=83L~%;-AEI+trL*DM%kCat zGk1&4P0k`oe{_>U-KK(})&1d@K70LJ>^1l*7}nvpgoVh95=V*qJN>4rpmkD@IGf)r z3x1G#~rCz?NUABo&}$9y{5qUB5(MX*&^S~4I8y8;W0QkUzI&x zEAGvnsr~g9N4K#adFJQsE!5Y>qiqg4ghbzRuij5sp7yICSV31a2p>(7|7|`y3eGRG9l-en~7Y`rrw%yFw;tus~by+X{ zg~8;iIbVnD5S`r}^6_5TokQ}*fkPX217^zf1laPrW7g*G9YOYeSfKtOmq^z2eaYDJ zd3d-r`zo)+B*Te}h1Pib`tt|zcK-G1jgmsUMtpoZBZf;UU_mk#4VPGzbm4c+$1^R4 zr{Zfk=Y&kJWk!p0JvHKFAOHdXWPk^68|pwwa#Tg0#MeA~1HbX25dWQLuwQX_JOU@y zn_XO#K8P9$O9f-Y)s=u_kJ1MPS9hi44Fvc61BLvjAj_Zdh+HhM`FrTJX`=uIc!K&g z2P?yy*hnufr< zHQr{ua3}-U--aH2G>IioD-2`mDI(+{>9i z703I=RjGf^4+%6393~_0H=9rkn`BS9t${<2o26;J*!xz+2o2+ftUj~h1HTcSS?UUY zjY{-WzMt|QkC^M#7L>LYkDQ6=+YDOmJGL>i4Y+B{5^RO( zgUYaNC5E)+a|drehIBaa26#T38mIQdM!~<9cq4+$d0b#T`;6y8cY-YhSvz^!{(}Cb zC-D2omr%btdlP*RxAU@|*(rK(dAJ;Ie;^-pGt@X+=DzQ^K<|KmPG~OK@~8ujG;0Ra z)7>-DzOv_oqr4e<(^5JZ{^S~oo-8uwi?CzssMCpku+3xb7lNei;LhBSuuy^*Ci+9KkY^dn{@mkh zDJdaTOjlL43L~VZ0&@fo@4D+W@(yT8nZ_dZ?AOR>B6B1G(eSJ61IP3b?-x#_9RUx*CGmNdhP|@)Y_~nSij%oNc z($`|K;$z`)_SGMgTp=%oe8v|^+)_pTfBs>YrjOup)M0=aQ{7)~90X;O+(X81D-mLc#^Ba-$bYFWAAPTDR{5=j*w%r7pAqBi$m4mO?G zHwRnKBk;O{f&-aNszpL|aI}5gu4O;zAExSU9^%~7K@?9RN1MJO`WK()*D5YsG#OO? z$4a;VP8i%(m;Ot=_8iX)4$fOS*;)}rdu5Z6Ecev7Fl;<`S{OQt98JP#jWh^#eUzJ)03`WCs;l)4Lz}|)<1?uF%s)bjvWv1hj8-6JCpWY6 z7w3(MVQN?OO7QWRmm=&qX<}#Idyq8$-9pf|`EAPf^;GMMQS5{mOU`UJsXe85$4AGf zyh3qBoQ;o9TpB|$tQtP-Edw1$vEqOZm>;Zrb1_E7aeK#&1@E;wZ^VP=#=?(t{tl)c3RAhPJyznMP{hZ%V z-@YWT{Kh|x92rb$vDk_>P?0q1C@Dtdjlopfv7?bmBOZH2tU0t@7vCY|XJpj4+Kzuu z9Nuyscx6*G9s(t50i|k5t5@m_uV4!oLj-#D9$#hbuzqwi@+g!~(C!a3#dK+E9Jz_$ zg95jC7>&9F)0cZgyO2fwLRUgFrP{Ah9xe6os!+$wDhh{rVfAPEYfGk`iG)O3=1;Hq zcPl(NrdQD5$Za6DOln{^MjXdRKRTHT2TZq~UZ>pWNC7 ze_E}8enBRj$~lg+O9vv^<=xU+W%FE!Fkb9qE`Qc>sAhyJwSs#ax+ilFTl;ILSQBEO zC6>uAPZP;{^%p^uj*8o>Ai!#dx?(t=+v6}C9K#6w20YHVar%y8#6>3CSkuV&w@c9K z4NR8!$>v{M{A=PaU((2@xq%FIh8IJR;%VDCM(kNk68+f zOZYm)webAw6T0P?l-i2U|MsO;_1gD2gqw%>1=aOi9^@|r@u@?ruX z3RNDqfz}Xrd#NDx0}o?FRFs3Rww27URd5J#25rPbLg=i;WgsAZN1HwhR*v=h4T;yU z@yqUrD}tGtFm(ywnk9|H7%keWekYlAvber3YE_Tnjy9~~byEA9YG(PeODlCkfFuvI`%NQs8^$;dI{IDV4lM%E<&Q9NJ7MHg%rakQ3b76jDy6dZB zP&W}!NsTI44tMvw3M5ds-g@-4YKFK&Ho!WVHFUh&y*52VICR$DJV%wJxI#9B;Plu% zUlX9lH=q=;Zt(r$`}+9$BKN%V{klpaU)lD3Kg!^CJAw+x=$~%O8(+)vCnhZ@--Of8KF>-H{YR_>1(Cc0R~-5={R<|H;$oPW2ZZ1Ov`$hFoU_^_$(aR8;Gq1>woW^lS*yJN8lh!bEKD zCaTcnrT!{I3%?cXV4i(dAWee-BIrzU+|l~9I563t!FtH3cKb#++kK!rR;Vv?k?Ki% znJHdEQH!ni*rxq=0`ktv&w&@1bIEWi#iKN$e8Kv(67jhdXO)IX2i$+?CNj{KJnZv) z{JxDT`*Sx_#f!1#Q)e4*-Qm9g}_^bh86-PN2A-X&0cOyD5j z+&F6k4A7hL7v2XP)~~oK*c?bl!3H=g^a@T0%5L}>sdnZqhb(dHp3=?Y3^8vG`rP`^o7H48Pv`66eVo?F#;44s_m0-j7C<08iOtgF2r9ZxKq zm}s(eXJGdAve)JTfu+Vz4U5_;wt{_;D7}K{CNX&80##hcTaF8P^s&w}39~2he|o() z_mW2Wg9D9CNc83H=t>oOKxnX0x4g!S=Y3VV+1E!QzFwWfFclaqcj=gwcSxC53&m?ABV<}&;Q;SU$4Dzpn68@r?l0~tw# zi7q`puL62uvm0C9CHbX9ns7y(@D_Xx8%+Mk7C2C|k&o0sdNeynv4piLHa`AocSrg9 ziVwizE6umHRN#bIK#r|#jV-fdWJNj=stbD8oJ%p;B04_jECaw82C$wniLmPvwTPQ0 zb;1~FinuP2e;aMyb}qC-LkU7N?I=-6dO<~^?)b`xH1S$YsJIrEZ^@3zSA>F?d1T!f z{`->)?e-C7E2%3-Kd(J%>J~StV+cjKCaRWo!92j%7mGGb?)KvA>cAI!ciKXi11Dxr zZ*mVRaNgM#G;J~*`qaVorG&D5%ktsM`TF)^Y~nTbY*axquT*8m$+x}yFyS^G*i zt@)`Ys*;iB${D?o`bUWI;`C1oh4lMsm0#N2L@+YfNZqF3A67rmrxc3X&4Fj&W60f~W&zG*X17QUZqA&)jT+NVv<4Ygg?<*E z|5Z=?@w>%-*8A&G9pJk{83A50FZib(3jdYgL8jsdJ}#jJKI|mG9icTo!yG|ab%a@F zmhjh=$qc-rI?k|N;FSQXP&T!K{G1$0|Kj!~zIzkF-n>^XU#)f9mA#GzQPO>2>AI*^ zQqG{`V@q0D*Yd_<7n~ zLqKfL`@uvIE?X%MW2qfNio9sl{j$(5iZTR3fM^|&mer>!B$(9|M+NGt0=3nt%>JI& ztUY8B2oK zhx%$?wQUuRt9QEsm%#}--wPAO^Rh7q#Rz{e*X!q1<}>|P>V0Rpu*Vb;-Ptn~d&rnq zsuwiR6qeh|o^qFo%~9jyeX*@dy+@%Ls~Mje5IotqH01jDkPD&iBCl|`z9sqza zG0LRZ!4!L!Vz_CxZyzS_lu?7TmgTk9X14?OlXii-lyryDc0$51iBznhG93#%av@Y| zGqaG{AAdopseB+>b`{|VQeulzaHsrb>dJ0@P+9nWa_k+#ops>sn^T#{QJHk&ymjcA zf{41+h{xLpbel4&C4BMu2TpS~&Gxjy$3*F0O{^=dV-~;*W>jVFl zTi?N_ly{z>0@Y3mlq!HtJ*S}zNO8R`)$C+J2do}g)731GEoH=idUog+{8efu+Uk?~ zI2_X`s`_*)7-`utQ{zswTy<6zJP7wuN1Ykm#hO;JXHsmdm+P?JwMMbcNc;nkvFKeR z-(>CGKCngs-DJIK9zpN0zAdy)cs)Hu&w{HbxKC2}dKB+>(h%lQGQCK2ktv<~C8RV> zzERkS!L}kroU+DTnJSJ(UHXcXlSCUP?%8sC(;)+}V*0pNpKY=0sDe{>h0TmqgM4sD2etbd-@N?A zi42q;x!7lclg(G%MMe8u)huboXjGTNa7=yW(5qHa2fNT-BJG4@dlYR7)*3Xc)lsgk zgN}dRetT3}urrP3;~i|n&!#~l1*l_grXN2Je+H~8?q!=U^#q-938aS}VJ-%i!Kq{f z*B|J@y7V2m%tw}EIVAwe>m-B9D}NqubQXTC*QPtxm1jPTeW7=L*v7Fsy4+K!+dkkn z+w)n>S4B{t^rESfZg$DNjO^)bd6P>pGymlS9ONv^Fz_MA(7^mm>r|=g`9#_{#POjoi zIicAA-P+G%Y%Wg^3qMd+@ZBZ|m!6-zPe>uH1Zs z!3-M4L1lv6j^h{MJ7aZMMb}3Ox~=oZUROnxZ+JR z_H?^^Nwv_=ITCnRI&L{yO9k6kOOi5|mAPX)v(EPkjs2U8BZc+iYc1B~*<3BmkImX^ z_|r^d;wJg(MY_Ow-BV1s0&hcbQta1|ir*`Gf4iEgj<;KQG+BFWwPXrhU(`A)H|6Vn zKihT;XE0>iyFvT#K`&kyj`C&psN$%h8`isOUiybD<-Cev!t1Ykl-xhL)oTeBN`4FHoAtwLbHC(3#ikm@IvoQNa!In)|});kuQvSNjz)}cSrw#`Uq!s zWw|vSsq}itbnj1AxRfUtR;x4W!;;hY*YaNtW?L8STmVC%xlITHL{DlT=JF4gP+}ih zWi%g#Sl=R#@Cn`m!qT&?wwxVb^(E!b+9S3!?W?V7{3*f7!^Xi{HnK?*(vAYFyo**e zeIsDOu936nuDtgHL{D+`VO0s((G~>irTvA4Ps_K-&W=HyOY=(gK(e^rH`f)7%9cM_ z!Ivyo=XNkiu@_vLiVSb%pE@Tc72V@hk@%oN4rYPo+SfU74-2mBTXajYOGop-p&8ndRROARXl1cOp9}p=eB#)otk-&hMa{=AK7l0?RCvHEM!pC%^1RD00l{SH zn5+lY7QYV$37nw)t6@}QU2rTeg#?moL$Nt6Bkib^?IcYS-T)3U3jNIfjJ(NL#!gr32IuK(E2B~2`sFQf^NN@}oZH_l&>YQ4k27D+N#gHP z57jPzkT*^RNw!KW&I4#N>WFz0XXP;{Hu&Adz9?`s4{f^FN!--&9aHd(#o2A>^S>7GMm973D zYD0J8uP#Fx*;?e-t?CC?QJ1lu%+i#zd3Y#&2X*tNFg{#ZOS%WRIMS%*$8V$S&JW`Y9>ZQ{2YSIV#z4tt1r8(`*F zLy(1BRg{MU7%=98y{qxrhsbhdl_<&ZsEizebh}jC)7TIm#Ck2N=a9*<`w^)cUcMU3 zM<)i$!BzaOA%2hB<35&~}8a}K%MAgU`4M!Q5wPtPdw$8m8kXhZ9-yi}8+e>Ng zx!QvX8~3hOf7g6=nI5(3l$hEVF`-hb(2jS_VAAz>rQuV@$5?+PZA4TEl88oB=sxLG zqN}o|Yfcw*aP~An+BYe)M6&?y*+_2}gz7jMNkx6XB<08gDmORvQgi!pf@3y(tcH4_B z5BjebD5fu23<*|Yz6M|J7Cv?Lh|NGu3f0aj&KG-GF6*a*e6YMJ$4W65_5oQ^eHdDtQ7n# zY;>KX6B64KM3e1Cnt5zmtzFfN&D9x z-Y{0HVQ8Jih3sXVTA|I|PGzeR99#gHzaoXye5VUZ)xPFJ~G5l zV*Ob+o-D`axEkgitpg^)C>=euq7x^)5qRrAbYcPMSXRGg8(CqIAL|CQtsm*;kgj2q zdbD<&aBqBq^ceZXkU?&tC91R;***TYnYPLy^ed+FsbmiI#K5IR5%KaUFBr$dlFx zNgm0e86xfkj!2jO_05D`-RYPq!ZJOQ5gCW)lRY@17TKngIL7^{hR~QxNNMryqYwPp zv$`8hstEZd1z?Mzgcic^zZkDO{{IK#om>Ae#;eHp55yZxgv(efiPkb+*3j)O0s_5K zG2FgUM69&C7Lu`c`bVU*!DP;5lxpc*x4AKK!r2<5EeZ?%j|g;$^a1cbGn14Nr=~oa z^NnOInV|Shd>k#**nAn5m1P$mkK=$pOuhfJAmEn3gR4P1!E^GDMg~ywWqk}v-7-!#9^T<&}|KTV@ZIFhAaGCjL z7KKzV?7VkTcCm=hF(DRi%RwNggfrtTRH2j za>G*64NNm&S3wG6!_yG+b!vo>I#448UT0)Ejt521)nTn1; z=}o0BzU_)weRv5tPTvL;2kf$`Sznyw+1FY5aFxykW&MNy+@3q$*( z)-l$@TM$E}*OPQ-?>Mo?lG;L?p3a9-?4iof`sZ0>Vc>I|sjSd3xjPF)%3TXa&}(HKB@vDQC{4+s0F!M$SSd$nm9x8ehv{2DgPgb?`P{wVf^y1yK+B(F&rx zU7(I?K3~0hC&NIKy=~QYH{@Meagtj(SAnCR45Gr7E~esoz+~?)>@0N3{UrDkvLEn7 zO80d7!5YeK{@%00c&5>fHqf(h<72JsYT3W1+MU4|HB91ov)YakKMrGhn#nNVOY}DYg zXLPac5;>~$NKK^3;Dkvctyde7v>AvjA?`qodlPhkc1{|R&cnZIcRUTQ9UGWPX=9tU zO$r(#CJ{PlGXYARN*xh(zei9DnfF8sXWb)bf+i}9iF4hjhy8;xqFG98bunyJqNj^` zqY}!YiOK6}Ca#B4m#uO-!I^9R!?l+f3G6muh_(3K|LpZ-wcj60K0W6tV+IT<-d-wO z5#o>PmzdCFxvI=S(4UFC%YLJBd$HZ<&zoR|=bPxy52HUlYk&4YKS^uPg(rWWPe78T zw$ctqw8tcyyu;AAk~UYv6>Z+3J>pdmDPg)C-%s%Y^PuTafiuM_n-Gb z@K(fRSNBQFs#;d=ur9yQXIGvC6@}}`F}7A;?7VLAdA+z5@bDPP2ZfyxM(S3rlYWf! zw0=(>zf^xaPOREbGw$+!x!%h2dzbq2veoJC{m1vusKB4cuXz8j!H>KzfaUvlnLy8< zGX=nT=ZpXQng7=tLVsu6u?DQAndYB8-HUp73U}h$m)<;Z7yPbvLg*j!8D{7!S+*uY zJ`XI3N5lNvH9Ng5Q#HRPN7dtJ6%vmEo zbjBu`#3pe?#^@^DtanUl+r(Zh1^fY^9ES z+Yw%WPBDgS!Fq6CFzx`AOK{x$IaluDucJe`OgNM8bTh@AW z;eqBnsCZ;~VT3zD8d4sR#gn<|*YgG(G(%&gn|05`S*a-9*~fz9BS!<+df8vVnC;Wo zTdor2VcoBH1ZYM>UTR8yah#~2(7Fu_c{rv2o5_mL2F%@ZVjTQ$xLzI#gkxJE3}{@#tmv}AYRGBB&t234Hn&^%uyE#MNj(=QY*hkc9qzFsPws-GA04`vxE1;8&$q4w3ZESn!caH6n55g6aZ)8`nVNb29M{y z=8hkCm}a;UQjyt5&j4GCnJ8@hc+JQ-HO1jYq-vior~26C6w3L$P5OIYFiT+-1CO9x(7tBKXvK^*5fg`bym)R(Lo;%e1F z4aszvRVQ4P=uETO7B>FRNb!rIeM+5=ngTY}c9sl#FmCl2$K>vn;vf=tzSCH&{0l5r z>1`5ov(KRp2GQ_|92br-PhAhgyzC~$Vi zx57~QM$gqxF9s`y5d?u%SwY9(SEyW07N*au!rws^e9(DT-`G)PeXK1*zXeXlzwj9u zX+Bfk_;G$EVTSr4r!=4-u<$ciu$#GK(vDDHnJ72Zg>&b*fx~nXWbCjY?nK#Ko=jDm zGa=mcm`ytVsPyOCH_=wK&st;wHt`Xgr_)HI_sPUar~M>&E`R`kG;m37kL$F)y&{Ha!4Qp+aIjnG({(bRR@#p3T6~S_wWb zG|?T{)7k?nqw?AqlfeH2_TJc`7!4Fpw)^RjXA|h-e0B6js{`V>0PVPj)B4)zpq(3H zR>NTzE83iPHng78&>O*}z6xrr8fS#xZW5a9#DHi+4CPi^nnx`NFJXk`cq)mK*wtRh zK%O&`$m8i3#7H`_-h?OBi%xO2dLGk1&y!pK7;amcYxIm+=pe$IeLLn|l5;z~^Q%U` zG#a5m`=YP*U8UQ~!nJB$vcCmgXfBSZ6=hclhX~(=>>GCiq%F7?V|jZM(zTE_Fo8=e zY7VXVD9!=-ZLpiLz^M_s?QN^{)K7?Xj}0eGedlWng6z}2w`IAQtn_%!+j}NR27=bz zBV#stJ$|n@kAZtf%2CTnRjYc5B9Bru1&=9=_}<-6L&(>{)p32ycpt;fPXCEtnuoG+ zUlVJ952OEOHo9}ko1T@U7G=#LeIZ37eCq4gDQ$hwb$34j(baW-KLP!>g#n$F)+*k6a}=!}U+O{{gCZ#X8Gr9Mz~Mg%*b_CdcQ|`wIw~tY}Q4 z?%AhO<{(^Q4S8Uk`f{DT$-jVro&Mx!?{0y#hay2&qd;*C$vNxG*`aHV{mQ2ANjSvt_zm&U zj|%4Qy)d4Jt5hwTlQFGbv^R-TG!!4wXscawz%f73di~84n>F4Vs}JS0{L2t#a869ihdh_YMY%-AJN5d%goXcPFg0wAUZ`Og_&sCHmqzwAmMn*IJ5;2Vf*SeozR=`8 z5#5cWI7DYL`aWHp%b<9sR$th1Oomixeqd@dZ--1xr$|6W+$5na-&wT>b3&y9@FS)e zWnOm4YD1-e5@M+d}sXCM?ih7?#2&Q2L~LQKj8^OSNH`kARY__j$0=vNk3 zW-NYcSvwDp6YvrKZ0c_AIiUAc(Z1OINcGh2ip{|>J=N{eI}^@4^;D!?RnuDPRbnr% zig5MR$v)T^X^;B~=JUIZn!v!6)u7qZJ=D?D#bBy8cvo(q=)ozOKOV7lOB1r_n1_+w-T5sPsJ z(}l94^}wt!&mQ-SrUa`fNur#dr4kTpbfMNPE%7W8YXZtDP8LJ8gth4kcNm*u);eZD z-J=HCJ%v^&*WH;|8yfvx`GzC5;TpeA1eQ(CMIDAk;v$CeU|Olf7GVgHp7a{TZ}l1$ z?z+;auE}o5V7t<72FQb6l2ij;lAc0qUXnZ+G`Tz&G_w-B(l3kUYhRL@@gbW-KuACF zE|b~4U6s+uR2`PB!yXEt5~83?d$k6MIq4*#eo0Tdb#qEX8w+2d*%qsPsVv^%p{}Fb z%n>uV-wK71l0Q!aYR_<$?n+1^GZ!(5p^f29kP;@kmlMQU05DBYmVc+B}uEneC8itTN(RjFbA5()zwu$T-R=><`yLsQA_yb{hn3pe0~K z@_a6s)XN}+8(Lh>ImruS-m5y%y=Jc`4>^oRgp;lS%X5O|B*NtqPZyT&=Qc#_OqMgR z4ofG&0wXbsN%OF&_yN-?ZX` zS=~vjTo2-$DNbkoOx>bbuNQf6zQBt~Ve4$^*X$$2p!HCozZ556pf}m8JhN+dk(sS6 zfBPIs9t2)6a~ZfSkPa%Bp?+Hal+y2wSCH+%_*AhRC#3KYM*nF+rEs&M0b?U9FcV3? z_)8^{LgH#rB&f~8H}3aoZZyvXdis)8&O7SPN@bAXfE^hzIfDz&$UB)}jP-033p}w< zRW+`|%0Rw88|<5_x6@%u>+lL}U}_|07{9Cr?nD+Rm~`8B5#PipFyeud=@$Mg5#x3o zq-1t+?966%d3=rMZXD-dk3SkZ!>R?1=c*b68$!^ako8fUbFm%j3Nhi#iHzo(8*UsH z5>L9+n?W6`zkJZa*SuVk9V*bj-G8?=7_!lhY89Tjf^c25_>JKq`eb7lSkBXewFx|6 zM`zG=v8~*h>t@Y|=Mu`oF3;f5TbQdHQM8!vnKVa0NIEgC+UHSRjDdLqLcKdwh!<7- z8DA#yJ!_(L4M-jIKuJ|}RmDjqC?xPsk$&vJ#leji5->-ds6=MlRfUG-q*DbUsAM8S z5U6Xe_FQl;u{Dm!*BH%jS$LFLZp&?7@i3ar9ORM#n>rgrwe=x+;@z^kb=f!$)zjqD zO&kW}=CNR{S>()>*3pnXLF5l}8}IEtX(!XEz!AGFl81y{da~VoF%T8WpB|%!MvsfC zw?EJWnfbWhXd}rVK7;YWdD(l7vOkCe+8O!8?CSd&La{TV%L8tg;(H`qON{~G;)P>( zh0d-AbM|cI1k`OsgL3lZWi;u6H|o5W|-mcP_)1*-H|` z7y9eIJ)pYeL3FQb0K?Y`Mn}(U7T`L3&#QJ;qR(AxGIuxN@WRo-6pFSv!f$Yn*$Aii zbdj9l6kF_rK1|&0xwfVZbH36W^^yNiIk_@L5DmVQDDAZ~Z7gYk=#uw1cE9(y^yq^g zb7y*2IxPTP&{_i%b0r&X(4tduM)hThkv%4gRjma{^L<%n+)e&mi$U~rl#Q(V(u`Ox za3rx+6pRkf@Ix5LEVg zyu1I+B6h7d+q9BnnC*kUza#8SP=CN@t~b~0j^PpS?qf7vrB(gfjqeVqq8SVayQ^k~P zebS+@AegKY@(w9zYOOHfAtS8fl%ce!ET;{7o zkJjy;PszcX1Ak|_52ya_;2SyaB%r143x5C%DF`PIdp<{i^Yxg+!8iHy$9IRGl>@@* ztq85Hwu4RR_eT96ImEFaO~gjP=O7zxH9N0Sc+|f^z9)gNx$)xS)<%jJQ<{c?#j&sO z%Bwp-|A6N3Lv}5`>uc`8Wi+u$Vro=|kz!jaHU=4?C{+y``8Tr==>g0FLh#7lW&N2GZw9N@V&{W*UIzM)wmQCFWKIm~i_fE-LaN0H9 ziXiPstL1_RaJJT*E!XO@6gAY9>q_@%|@;#?%FdOgSR2nCoO@ zFS-8de*?k#_%J-$fVI~2onWMIf$Sumq<);#BR$=w&pwS4ZIoG(FwC zAU@u2#!=CO3!F$03HRXr&fLB)l+tXkI)|##cX0J?voY$8j{2fct8dY}O}JEMG}X`eGE|1LKf4QK zmnQP8xO;+yH-hNaX&L|uC=Z4w1ZmVyDr$$TIt5U67zcgsA#~_hI`A`fd-H6+Ad30y z-W_c+XhZ6sUR}Gg{lTfZE1h|nLrlHB=q_@`8hwyhNtnwcMZ>wqFn%GVL@e0KDuJ5? zvLMKw>F@m>obi&Rmk$u2y+LU9`A%#Pp!RZYpg9ksw~Fq%*gy4;j!o}ig%}7Cs^hno$=P7$9VP0e2B@A)*L5JtFB=sv7 zYqF|tDx=?dbEzJ8Xh=B`5rULbM%pI@PN;Lv4>Q%QVIPBrLuCx22MxVJZxBV&*|yJ1 zmv&|Na@#d+kjnu;LBM77c)+cj49Whc|~FC%`#i^w{&E zaP{)#)W2{{IJOh5w%c^P2K)R~J0+hu;_GET*c*^y0^BkU{_(AzvfrAxpFmoZC~zZ; zEq)PAN7#%mHm?@flGj0(95O~e)5POErBL7-+u6m-U8H4d0h6j0^B8n)0ywqS#Nr2g z8c5b$ymR5C3%lz6xDp@E=%R>BNQdMN0X8wqo&0K7Lmo)G$JMi zWqPz9Mi}T)x;Z!ftWhDqIFd%S1iYi8BZ0A~vArO>wKd3?lzv{$63vNHHWcL&Z-ABH zuUSl>>h@dM=!D5cEUb}Fl(+U%X~z(WmiG0H6nUxO>o8D1Z?Y>4NKLkMl8rgGX&{^3 zlw8YPjGD$jwo$j~AW?{3;z~tKCE%xnWO#GFZS$WgGT zTAwE-I|kHgzhNE6J0(4=qJVQbvz|Y_z}L>CK96|*kF~=js4*%5ZQvgX`UHUuqTXj-ojT=KNZ6T15+|+ ze+x|rYcZ$M!R3)e8}beJj3ZpjgiM$3dU>7>cKQYd#QLt;0i&*y^>iI=Ps<8gHo*L}L86IY(o}4PN zB`iSU4vb!siBLpM227;~hRgzYodw}?bamw|EtP_Df9Jxq72n$-yI?0dNd4M}jVBkc zLP~X4hUI6~@crY)Mps-qW_|tB7rfFi`V&4Nl}}*MIW9h}0JhsX?n|>9ki!_xnL-aM zAobN}W@}PRK^1PueldQ?=L^eL8osg?BI=f;*i))|88fIrmP@YgXaLsk!GIfuVAb0( z*rw5?DS)M^Qi{cpm*~J&Hpm=e2D5$2a z#9v-H*_b!z!UbKqz3Yy86_I+C^z%n3xkpV(eq$38xHl}n!_eJb#7?6(ap8k?r8RVI z=en;S;+CsCQ5<6Sf*@@7WH%|l)F}cYvMO4AfSQmG-N5TTIABQ~tpRN@UejjZ24<>c zp?f}M@WY+UjM>-^;*>oc87#~j4{;YcGWK;YOthmQUU^7!iZEt5ZJbcFD&(g%^;D%t zHVRQzyuxP2W02@C_`I_y`3f2qyXa)HD#0}_Zm%6<X?Ilu5=w@zGCddj8PX?|H#2!ZYRm|oCs97dI!T~jFX`rqUf zze5xPWX-Sjk`0iR6_N(9O+JOU6r{t|i4lE~Uxjz;*p*1$~0KpbXM%kKFoY z{3%A5qCGuc*tFj#HzjdB$=m$2I}x2q!Qv0*$;v-a88t0G82QQCpGXnN1HGjcTz)CY zd7*oyk*wlMFiS<8KZgR^5tDYLk$N8YKu;*{-0X~hGxbxD000aY_Gaun| z(`4q1z&ss|bPlAzfbEV5Kj~xIVP|TkMME!iG3D%?tAn~>i(tOF$Uo7`ikY>kEpCUN zO%0dqKhrnmQ-I>piBUw)-;K15KkW`ZA%~xDckT}F&%c63iw%QouCaUZhdFMZkW1nx z<8g!gD46=BO9=0#r4m8sGqLr<*I)j!-yy(ZsAk_ms+5dU2tbMk48u{dFMp*Qn zkqI@HX5#%lr{jw}HLjk~zr=d~BpO5%HT)_Zom}-EeCW*El_hyMA3;f6rZLGQaryCr zQlr~GrP3VENfl;^)@l*WX05HoV>fEl1!KLbBY1eqL6K#WBMs<7npFJl1F2e1;T0r6k+VOeI-IRM++P zh}*JKLFTi8svuh|b|(Q-z&OPPpcS#3=;GzrH<(D*xRTrKbOmDHg(>Fus}EN0=&)Pn z_TY4)T+jpTck4o_Y=kR4KIN?>+lJ0Nj~wFE??E>F>|k?tFTWE^Br+yUr?>2K^?wa*%L6oKjE*Ca0pqH|iq_ykl^(YATwA$`8g^u6GrA_CwFqSl~6h1AY zm$DVOk;Q$JwG^lOwMS*O9>W+qzmW8j79Ok(rhRw2v?N0=b5-)dCV5Fd+P`>G85u0fsHy&6Smv zkOfmmG&?>c(?{a2*P)wTv&qZlG)`vz>SMmF`k>;pDJduAQjDQ6>UJ8cp(x`RlPK?_ z{$|XXeVa;MMgdiAzmV&=$ubd!NqUautFSsSeRjG(v@_H^f^@?%@(~lRH)h%4bN8Y^ zmoRW4SvH*%3w69GX^>Osy02=xQASZ?rJx+RJ`B5jZ0%HKLYGVib&HkMTvzXDgv~Ec z6MF~LUY?68tN1Dt>zuF@5ZJlus+i4;#MQy3OhV?45%|N!e11&`A^GI9E zH>&z))g7zJfv*B_Vg8?bHA2(rWmAq|D8dHr)Uqt53Ch|FCY9LR3!@ONQbSkDi8l1A z0$TRL&jdQI-cIU0)ocCVIC)ztmw6u&jmD3Azym5mF7sSv<2`;4W5N*=2<0E?Z?fsT zk^I1*D6`>)Q=w1FNS0@DY86~bINwV`d>@N`Jj0u{KR12IaPwH1W##?ez4_yD_g6Yc zl!5xIkYn&jg#b%Y@=_Xfbnj+$`X|~``gi&XfQoJR&P}HW|6OeRp636FeB7}_WX_uz6~YWmK%??w-J5;m=&@Y|33ObNUL+*^|fk$(yRA-^EYs}lYI>4w&&;mMLL6e zYfG&?{LKu;*VBc*z1^&R?+&Y9vBnlh0nqm`^}oXU+vXdH8-L3DUNrF+1Lt~3*I*+N zLSt**Uvv4Qz&}fUe~E~wrqC^9MC`9({E!FA#O-b58M*l}6PN4{eN3A6YmNC3iyQ}s z#Y;&%izlTV2YJEbYLhUTTU1dv-?gtDt10e3TdUemMnH zIQJ4G5J1NgEnMi^#kXXO2!ll$N0ifc$Ju|l;+vvKie2A3VHb|$V_523)!DT5H9vSH z)wQ2NuNN(y{@K4A1Xac0wQ?FI4r_V2C4M+gj1vrjo$G;h_2*o3i*JF)=+kxNk*j)p zUR}@zloE@lx8k+zG-w9N@DB5mn4y2v4sM(I#e#~S>}p^1TUXV1IWFn;6o-@v+7Mi} zIf1-+l5S({p1fk^+n@liNojEvk7Q0n)ksDm{$VOw(Y2or&ausKnxe$&`&#ayW|NhD z`~{L#J3RVDREGXjtoBx(DK&~g$*Mgan-_Q`DftmrW9)C^(K`(Y_LxA%5j4w`gV*cY zEduKc1}&*!5_h95mjn?GSuzmgAU|YL_snA6hOse;{kdXkbjcXTKmVxoBEFe<;Nkc= zBj;QX1Fsr5B>5e>TVm7SruND*BS>pZU;2|x;AjTMKngdDcH#uDuCQgrCcmb%bb}X| z!z%4*{km?TAuIccM8J#QF=O6&nZ+)3v|=N}Dxh7?{C#9P&Qv^D#X#I?STp1pZCXX{ zy&?8-ri6g3EqHFhxUOiA_qJATpy{2(swiwO_xD9+Qg$6TaYh?OM0Q5v+i2YOUOr>< z7LdvWAkt|}qK6|d(>^5s#!QvA_{&!j-@G@D3}SgaF&P_R)$~9&WNb%&vHc=5sVS

%nSS&xDrA>nMr#AM@y0DiC`X95>Zw-g8B*h4O4EwyoMHt>09gtl&?jgObK94=$U z@^LId-bymF?LIr&RY`7@Y_siRG}jEv+e&nf5IFY-h3W-#>}o{b7wqCOTM<5 z*Ok65Tm*-;$O(=eR&$Os^Pgq%C^ZDlwao8hiA6EMhWFS8)ulz!{pNrq1TV=2QMyK# z60q}qy*zW>``)Q^2ctf#m5^KwWu&rIZI=uUxvSV~J!zt{b$a&)1TycU$GYqw@GS5| z5>Z`)AM0M?%bAgH#|GQv<6#e*K3<4Qx{B&Iy13X*TwsQX+kk5ZIqxS>E^ag2@frOr zwcuXhI}Sn-quWP&K`qL3UCK0yiVnEB+s!Sd{UdockM>}226yqkzR4WyTc@(obRArb z)5h~IczK!|PVC%@99-1uI+3a&mrH+@DH!FY(dyT>)Eb9xzkPFiXCGcXzxWw$|F#H$ zHSya00qM?qGoD=PLE-7nE4xS8@!I9U1g#4(_KXCgDV(A-X5cl~E!wmLV)Ql*dlpFTHJ+J7dFT0krug zzdG!|3)Z+=PgxOIx^e*(Bku59rkGBBgiTA?HWiL2sL5SvNa9gvNROX(W$Vz=b{626 z;qUaXKabws6)Voammi`K8U;d@#FM2%EotRrwB0zh1POpfV%HqR8QwaOVt_Jbh^x$& z7x;1qGC{AGaQ{YEooVA_87!<%+e5B9IRHx{Dz*j&p`gM8e=MVfHOUd;|{{`ZP^}SdP(o(GYYa@%VD_`}`dAKq^ zbH477wZF2w9o`)mnA&2*I;tS?IiFm~E}tBav8RGEn}j#=2D(|*S4rz$Stdig^2ZGAj9PaLK$CBa4^9Jj4mSjvnKbPt`$v;UyC70T z81Me*R7ylF4XPN0ok8W*I92NQa*_f#lC6bSS;N465{M!rZF#cLKdkl5h)Dn18;K`; z;0rR7$2zm-bHgOp%)-F^mY;Ja4tI_uekJija`s2=?qTFMe9>ATenAt?tD_6}9q9NL zGyM&!8@8{2F37&S(f>^*r;A0hsjx}^O(w53t(2#zxV{#EWvh~P{TuYTo1SyJ$SC6oEY z+y5h(3@i=FIoHRO)@C&Us^^+Dh#EBXy0)GlPP5xOPgL3-;L9b_L_;d)L-`BoRk0zb z-iK0??NyPPKzfBpIuLYx5&a19c`Be_WxSW{#xBD6LMjp5ur z$7`p=+lPJJ%$@&9CA(D*yu%_;_FFtAfS1sQjnaN(j-J8L6E&)hOwq@(LS$S!fDTW~ z(!M%kBZY-oJ*wVkh!*j1Da4z>X$4W31V-$7)}VIBg7lTB8*femq^)>ar(%G#wSo~j z8$P({>?rYTKS`@fPJe&JiL5HT^@s1!VW?f98i(x#k}a0i2nuOLbLhiF(vO!ikb)i? z8Hdg5uSKVOPA;?Agb#!fHWL|}V0Kj0dA3wsBvt|P25_cq5CB!}xC^|| zcA>gp<|tZ%e*&$w1xa~y1y)~8h4Rz<#vTOfM}}zBD%H6jy!wA(YSisNOdXAvi8{j_ z7@Z?mYFx!F4Uuszim)X;AOz0qxw`Y9ZvXiCn&3^Ly4=dQ9dJPZ(J~sl>*EKq^Nh(; zYVVEv+A_*s3uv@C`8q^zI6J&BHz_x;ZS`WGtZc3_^kQ>(86K=`NIck9u)DCe!-~|6 zZT0T0E*}ae{gyR`2LH-i$~gw25Db(K;kQacXbP&EL5+b}5l&L%v7H2fJ7gfG(FCOyA;T58KC@6o$y|A+N9`ng1x0-jC zjeBQy0a=6HSXPMKi}J|<%t2cnDjxRmd4&PsrTk-xM}A75JK-X>)bKEkjybDX-;9NA zRrOQSelH=H&RdnTL^rB37wjkck0?}N*S^3azVe~|c=bBc z{w`^p5dS^Uw0i@1MaDhli_~LXADwJPahdXSLbJtaQLF3*nU;nHzD^dIUpb9mwwB5l z&RS4K3411)v`y|TXYX#IRu-{#DRVP?2$rgdG=xZ>43PXTzj{V+mi3S{!gC82nnvp0 zrBWhK0=cG&e(lc2FVd+ZZAU}|R!$k>7zZi=#kJVQl+j^30FdcRT1QJF=TEtW0uX3MgFmGT+lTT80@(#(Nnd!<(G7q!$E$!#hoEZz^gjSb z2LPh*k|?mH$yG(^>=KwDI=>Fp&5AaS_$wnc0y_HGO3oSB4Z>G?{%TroIg=2>7LQOc ze<`Hc|1S!$ zM5pFq-zsaqNRo_Oj(`yP)P4N0$M~YNsnpU|jBdS{-qx%GeX(b2mWM@i!bH}P3+c9A z&X2=#0Mn!ED@X?tpNrkDtE;~xw;s1)M(_?0)onW8)&Qb92f;f%4)4$ycr#X`)Ch67 z4PmaFay`zLjXmcGlU-;o(+sSsi#0Ps>zpVpuX9UpzTY`4unk|~G8-Q#h-)Ytp|sm{ zpmzR|6HmJ()+J6PLEr<~`I*iO|C9K>5MvplgmRh9Ki-}Jo+a$%{BOTb#fD!MM?_a6 zZK6o8|Hf@Z*XyMu|HGGo|KQ7e!~caZl;KjG|HT);e$)Tvi_g&ihA-OQTca`H>usN- z2>%Ti+osi+ihsdH=ZJ8xtOxqxUvL2f0>FjC@E^F4PyGunAy@#o4B)yUT@3Seoa8d$RELGvu31S6TWLGEFm1RVTt(nHIy6ugp=$Ecv3f;JK z07^C#{&^e?pRq}|zt~3H|9E&-I`Is;UHmWpdU#BgjJ~PHE&x0{H0;%`@AUl*%m+Y| zr2lw$9<>ff>mWqYO7?iAf;LjNvG5r_C&2EctH*sMyAqhC|Lrui`@=XI0&p4{a$qju z&co&+n-Wv){^K+}W19F+r=cQuUm!$W{gA3M{%yquZYd(^$mj|LpAK+uEIiXR_~j&4 zp)Qf-zQ}kS040F$MP$69KRgK#%D^kZ02uNWhd^fSHyxfN$8RwFx1X7z4;%1+Wg^eQ z0+a7zOZyO5#&s78ex~dbbjOW6Ss(xS46V^t^gRTj#f!peDNUJp&3>)^%d^wWZ02TJ z%#@)Q{#j0DCQmXjrx@EKjh}3xh6S=%gcY2&3#~~=?2$_&W?uy4aJG#3((oTp5!?U2 zhsvzS5~f2uZk=K2uLurAk)P*0*89dfNFPS+6k|U)Z18G_cf4q|%S{IXYdCG%E!?c@ zA?m@tTZaqq!AOGjN$dM^R6cLzc8Ds{TXeI>S~G z?LcW&CXwVIcCx0#X&V03$f0C4{%YiHTmz-)hE&>clfOqz#f%kMiSx>xkKF7qC~oq1 z);)$F<{8pLetB7b%r!A#f$~*@kqh(zgX5G%yRr)^$OJ@xN)QDq;Z%`)&VR=w3${IY z{}>)%SL@KHwK0DF7#h%`7#Kc&kr;tbd{xLxyL4xgOTtv^Lyc_KvCwgv+(4d6tp_c# zeV(6?z^Z7gJlln1{@{uEvSk$QTQ|9^&Xoj7mtDY_bdte4OSf#NhF!rJKM zc0nEk^XOVO_T+s9XyhgO?lwL*u{?Eq2kz$IFM3wkb${$%^TC8 z%iUq!`24t|U!#^r^;|nQXXYL8^jRqJg9&2+b8>wIz@N%ZzYn~h_S#^nt}RmLm21_v>GyX)N^ZG?@e2AS%c_{v8{{Qfr|c`ViV z0}2|GwM{E<6CWLzt9q^>PH72K=HEdW z!G0>o^pIr03qwUg6KT$Lwz4LCvCaJ4Z&|ie)XRw6IBoI!_JK{SY#eVFz7ZD=K8R}< zJI|5v5{9__7ELq2I#0~{>}p>GrwGv_Cswu1=QPav>?Q5C;4zTQP-?%eSzOe=l% z;sryJk<6+Fy4LkPJz=i#*-thi9m zRgr&Dej1{@*nkA4kzU?s?ld*|(@AnM|0 zyO;U$ZbA{?ahB{1<7hJd1jl(HR&!J@CF~!N5TRvTt@Ld(c=%pyScMe_MYrh~%R}xD z5Z_@YhN%#L?!YftF69trn}hT;K#+296!8zAbrGXjC_-(_8Anzlw8@6*zAj@Dc$OBH8#~kqBt}vi%EfDk zMh7V5)0P%w3@vqoFqq+st@7G}=N3YUFmp0KGU|kAnnYgG?m>)RiNp~Fa(@D57)PnI zwU-pks!vi9Grr=iB2e4NF0(Hq2T89@>VP0XR1a!dROTm>52DyBbP*(D#t5nVSiC=0 z|0R(B#NC(d#7+^S374O9lL%(-#73RL8=@QGmj!+yxRDsbm0N{?b1kxcH+UrD;OeB1 zI%Cdh_o4`zPgF;}seut6S$_+Gt2y1pQHx7xvQ^eFgNgX>6mO%|Lldl?8;Jq>!|kYkao8qD=GR?xp=L3aHOe1ALm&$_p4v>@%NBX}k1Alg!i6dL7 z{`$hyV*ZVOxh;o-iJ;HHXF%(N7kZX)(ndt2R)(gWT!wA%lSAb-<@?nQaO)<)1; zS8uAx#Yu%NBx5cH*r^+~qb@Y4MO;Uw*OOu?W;u&F0+82m$>R^9ulXG`FLxuKwq-d9 z{yQz`kw)Ngf~SYFy7KONfYK{Xdw3fll7kU8nz49s-Et8<_iYjzZ{eS1R#_9xsj1GcB{|xHyYvwN#l7?-Nov+ONBL zN`VEO&0USyghVYnMiNmG;6MF8TJ?SDx}@8B%UXnrX6-#0Wy!BH0JFKSWZb(VKOoW**+}C z19Ew2Nas_=+7!NG3nVLgga$^3)GuI*6$Vu{%J|vG`+1Llkq-K|749$6=T7K%EGzLf(fyxS5NGL?PNq(SptShPZOrO$h@VMMKMfAI+@ zl(;WH27I6EtSoISYFk<2ixr8<2-t!rL=q>Q6kmNQD;|291|BlGMnfdcwBa=bh~ujx zYS=8erNt|C0^r6sK`BtgYVkrH(w@)t`^H9x_BoyxuUh$R)YD(4HzY z3O8bhNYh0vj2>jxWv_;#`3G*m7c+b3c%ta7G4> zknO);HSE%}yBwq7ceNB;qv(1@;!v>iGn;MtA8w=_EGrs9;XR}_e9Bu?d*Ea#4o2FE zCecfof2jT7(c_OV#cZ~GZwQsT4O62tY++1AIof12gI)X~ofT9KQOcaI7IWS7Xh>l* zt((2N=5&kBf!!tgg6_T(;T7EpMjFBPZI2hub7G0E7c?c6jgx@FQZz6}Jidc4AUq^9 z=|jFdm|JKgl%(T!fJtpF@Nm9XmZIpPHDwmxR;%gB3$nS6!NKGFB^3l+rQ(cKgwh~H zu6WB3vDk@vo@03Wp0!RW+ky$_L)=qq57I>pJz3}&9>0;IENCW*rJ~I?y6XhfA=`mJ zF+!E#_a>-xKT!YdVxlF~kg23m+xt1$I6Q4ckT)~dG#!$TPzF{|x{uhYiIW>|);Vs8 z0T4Tx>F+>O4^1^Y3H$8wR1U%H0^e5m;Dx6UlP;eW81hAU0_{+HX^^S=uhU<{E5uE& z+a%VBFW+ab1u4`CpO4GQ+%)p2SMs8vY}U|8S6~tLOMDjFXvO0ac&nfPM5GkK=+&LRR0|uDD{0 zSA8~qcf^-=;8!Q)|1N&iu=#B+!ZC+lJ2J$_q=5dufnRN|f?wmEsXRjZ1ycmA6AGXM z#7_;bpQ$EEx41Ml9ovNhtIiG@5ieI0%AVVAtFgbg=I273&xpV0ii!&56;%mNOwhDF z=V1t%sA5Ze)DkGGr0c7v4wPTDTElmNClWYgYYaN31Z+5JI{CS~4(V?8JaO=MQf(2n z&YD8*4}yl22b~s*z$f~?94Ce%rsA=-+TZ`6S3~v%fgs19N^?bD5w$eQ#bGW#CiYN~ z9(&v^bImP9tJ<;uQ_}h|g-Ljf;CYs20ePiUJ?{VZu`>%|pycoz@89(r9Bw(t3!!LU z5V7s|;o$dnG?=Bj^oDt}-R=ESmgl$d#}}}eeb0aA^X@b6>-MWm;Ln4=pR@QI|M$hb zKeyMPe}10ksZkLEUDkXndx!F1UsSIdN`ek|SInGE0q{7`EhJK&I zn3@4l^xw;0&$V-rU7($H0w-k6?6 zLRh@CoStg159v!GWctJ_3}F5Y^pE$z+#Hns4*utDCtaHCp22>{YQdKT3O1c^Tk-4J fzYEKojnCtDch6<_4=fPSpFbhT4Ddj^FhKtcZyF@) literal 0 HcmV?d00001 diff --git a/charts/fleet-crd/103.1.13+up0.9.15/Chart.yaml b/charts/fleet-crd/103.1.13+up0.9.15/Chart.yaml new file mode 100644 index 0000000000..11314116d5 --- /dev/null +++ b/charts/fleet-crd/103.1.13+up0.9.15/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/release-name: fleet-crd +apiVersion: v2 +appVersion: 0.9.15 +description: Fleet Manager CustomResourceDefinitions +icon: https://charts.rancher.io/assets/logos/fleet.svg +name: fleet-crd +version: 103.1.13+up0.9.15 diff --git a/charts/fleet-crd/103.1.13+up0.9.15/README.md b/charts/fleet-crd/103.1.13+up0.9.15/README.md new file mode 100644 index 0000000000..2452ab2f1f --- /dev/null +++ b/charts/fleet-crd/103.1.13+up0.9.15/README.md @@ -0,0 +1,5 @@ +# Fleet CRD Helm Chart + +Fleet Manager CustomResourceDefinitions Helm chart is a requirement for the Fleet Helm Chart. + +The Fleet documentation is centralized in the [doc website](https://fleet.rancher.io/). \ No newline at end of file diff --git a/charts/fleet-crd/103.1.13+up0.9.15/templates/crds.yaml b/charts/fleet-crd/103.1.13+up0.9.15/templates/crds.yaml new file mode 100644 index 0000000000..d42811945d --- /dev/null +++ b/charts/fleet-crd/103.1.13+up0.9.15/templates/crds.yaml @@ -0,0 +1,6859 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bundledeployments.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: BundleDeployment + plural: bundledeployments + singular: bundledeployment + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.deployed + name: Deployed + type: string + - jsonPath: .status.display.monitored + name: Monitored + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'BundleDeployment is used internally by Fleet and should not + be used directly. + + When a Bundle is deployed to a cluster an instance of a Bundle is called + a + + BundleDeployment. A BundleDeployment represents the state of that Bundle + on + + a specific cluster with its cluster-specific customizations. The Fleet + agent + + is only aware of BundleDeployment resources that are created for the cluster + + the agent is managing.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + nullable: true + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will be + used if true. This will try to recreate all resources in the + release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + dependsOn: + description: DependsOn refers to the bundles which must be ready + before this bundle can be deployed. + items: + properties: + name: + description: Name of the bundle. + nullable: true + type: string + selector: + description: Selector matching bundle's labels. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + type: object + nullable: true + type: array + deploymentID: + description: DeploymentID is the ID of the currently applied deployment. + nullable: true + type: string + options: + description: Options are the deployment options, that are currently + applied. + properties: + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + nullable: true + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will + be used if true. This will try to recreate all resources + in the release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + defaultNamespace: + description: 'DefaultNamespace is the namespace to use for resources + that do not + + specify a namespace. This field is not used to enforce or + lock down + + the deployment to a specific namespace.' + nullable: true + type: string + deleteCRDResources: + description: DeleteCRDResources deletes CRDs. Warning! this + will also delete all your Custom Resources. + type: boolean + diff: + description: Diff can be used to ignore the modified state of + objects which are amended at runtime. + nullable: true + properties: + comparePatches: + description: ComparePatches match a resource and remove + fields from the check for modifications. + items: + description: ComparePatch matches a resource and removes + fields from the check for modifications. + properties: + apiVersion: + description: APIVersion is the apiVersion of the resource + to match. + nullable: true + type: string + jsonPointers: + description: JSONPointers ignore diffs at a certain + JSON path. + items: + nullable: true + type: string + nullable: true + type: array + kind: + description: Kind is the kind of the resource to match. + nullable: true + type: string + name: + description: Name is the name of the resource to match. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource + to match. + nullable: true + type: string + operations: + description: Operations remove a JSON path from the + resource. + items: + description: Operation of a ComparePatch, usually + "remove". + properties: + op: + description: Op is usually "remove" + nullable: true + type: string + path: + description: Path is the JSON path to remove. + nullable: true + type: string + value: + description: Value is usually empty. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + forceSyncGeneration: + description: ForceSyncGeneration is used to force a redeployment + type: integer + helm: + description: Helm options for the deployment, like the chart + name, repo and values. + nullable: true + properties: + atomic: + description: Atomic sets the --atomic flag when Helm is + performing an upgrade + type: boolean + chart: + description: 'Chart can refer to any go-getter URL or OCI + registry based helm + + chart URL. The chart will be downloaded.' + nullable: true + type: string + disableDNS: + description: DisableDNS can be used to customize Helm's + EnableDNS option, which Fleet sets to `true` by default. + type: boolean + disablePreProcess: + description: DisablePreProcess disables template processing + in values + type: boolean + force: + description: Force allows to override immutable resources. + This could be dangerous. + type: boolean + maxHistory: + description: MaxHistory limits the maximum number of revisions + saved per release by Helm. + type: integer + releaseName: + description: 'ReleaseName sets a custom release name to + deploy the chart as. If + + not specified a release name will be generated by combining + the + + invoking GitRepo.name + GitRepo.path.' + maxLength: 53 + nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + repo: + description: Repo is the name of the HTTPS helm repo to + download the chart from. + nullable: true + type: string + skipSchemaValidation: + description: SkipSchemaValidation allows skipping schema + validation against the chart values + type: boolean + takeOwnership: + description: TakeOwnership makes helm skip the check for + its own annotations + type: boolean + timeoutSeconds: + description: TimeoutSeconds is the time to wait for Helm + operations. + type: integer + values: + description: 'Values passed to Helm. It is possible to specify + the keys and values + + as go template strings.' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + description: ValuesFiles is a list of files to load values + from. + items: + nullable: true + type: string + nullable: true + type: array + valuesFrom: + description: ValuesFrom loads the values from configmaps + and secrets. + items: + description: 'Define helm values that can come from configmap, + secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439' + properties: + configMapKeyRef: + description: The reference to a config map with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + description: The reference to a secret with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + description: Version of the chart to download + nullable: true + type: string + waitForJobs: + description: 'WaitForJobs if set and timeoutSeconds provided, + will wait until all + + Jobs have been completed before marking the GitRepo as + ready. It + + will wait for as long as timeoutSeconds' + type: boolean + type: object + ignore: + description: IgnoreOptions can be used to ignore fields when + monitoring the bundle. + properties: + conditions: + description: Conditions is a list of conditions to be ignored + when monitoring the Bundle. + items: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + nullable: true + type: array + type: object + keepResources: + description: KeepResources can be used to keep the deployed + resources when removing the bundle + type: boolean + kustomize: + description: 'Kustomize options for the deployment, like the + dir containing the + + kustomization.yaml file.' + nullable: true + properties: + dir: + description: 'Dir points to a custom folder for kustomize + resources. This folder must contain + + a kustomization.yaml file.' + nullable: true + type: string + type: object + namespace: + description: 'TargetNamespace if present will assign all resource + to this + + namespace and if any cluster scoped resource exists the deployment + + will fail.' + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + nullable: true + type: string + description: NamespaceAnnotations are annotations that will + be appended to the namespace created by Fleet. + nullable: true + type: object + namespaceLabels: + additionalProperties: + nullable: true + type: string + description: NamespaceLabels are labels that will be appended + to the namespace created by Fleet. + nullable: true + type: object + serviceAccount: + description: ServiceAccount which will be used to perform this + deployment. + nullable: true + type: string + yaml: + description: 'YAML options, if using raw YAML these are names + that map to + + overlays/{name} files that will be used to replace or patch + a resource.' + nullable: true + properties: + overlays: + description: 'Overlays is a list of names that maps to folders + in "overlays/". + + If you wish to customize the file ./subdir/resource.yaml + then a file + + ./overlays/myoverlay/subdir/resource.yaml will replace + the base + + file. + + A file named ./overlays/myoverlay/subdir/resource_patch.yaml + will patch the base file.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + paused: + description: 'Paused if set to true, will stop any BundleDeployments + from being + + updated. If true, BundleDeployments will be marked as out of sync + + when changes are detected.' + type: boolean + stagedDeploymentID: + description: StagedDeploymentID is the ID of the staged deployment. + nullable: true + type: string + stagedOptions: + description: 'StagedOptions are the deployment options, that are + staged for + + the next deployment.' + properties: + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + nullable: true + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will + be used if true. This will try to recreate all resources + in the release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + defaultNamespace: + description: 'DefaultNamespace is the namespace to use for resources + that do not + + specify a namespace. This field is not used to enforce or + lock down + + the deployment to a specific namespace.' + nullable: true + type: string + deleteCRDResources: + description: DeleteCRDResources deletes CRDs. Warning! this + will also delete all your Custom Resources. + type: boolean + diff: + description: Diff can be used to ignore the modified state of + objects which are amended at runtime. + nullable: true + properties: + comparePatches: + description: ComparePatches match a resource and remove + fields from the check for modifications. + items: + description: ComparePatch matches a resource and removes + fields from the check for modifications. + properties: + apiVersion: + description: APIVersion is the apiVersion of the resource + to match. + nullable: true + type: string + jsonPointers: + description: JSONPointers ignore diffs at a certain + JSON path. + items: + nullable: true + type: string + nullable: true + type: array + kind: + description: Kind is the kind of the resource to match. + nullable: true + type: string + name: + description: Name is the name of the resource to match. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource + to match. + nullable: true + type: string + operations: + description: Operations remove a JSON path from the + resource. + items: + description: Operation of a ComparePatch, usually + "remove". + properties: + op: + description: Op is usually "remove" + nullable: true + type: string + path: + description: Path is the JSON path to remove. + nullable: true + type: string + value: + description: Value is usually empty. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + forceSyncGeneration: + description: ForceSyncGeneration is used to force a redeployment + type: integer + helm: + description: Helm options for the deployment, like the chart + name, repo and values. + nullable: true + properties: + atomic: + description: Atomic sets the --atomic flag when Helm is + performing an upgrade + type: boolean + chart: + description: 'Chart can refer to any go-getter URL or OCI + registry based helm + + chart URL. The chart will be downloaded.' + nullable: true + type: string + disableDNS: + description: DisableDNS can be used to customize Helm's + EnableDNS option, which Fleet sets to `true` by default. + type: boolean + disablePreProcess: + description: DisablePreProcess disables template processing + in values + type: boolean + force: + description: Force allows to override immutable resources. + This could be dangerous. + type: boolean + maxHistory: + description: MaxHistory limits the maximum number of revisions + saved per release by Helm. + type: integer + releaseName: + description: 'ReleaseName sets a custom release name to + deploy the chart as. If + + not specified a release name will be generated by combining + the + + invoking GitRepo.name + GitRepo.path.' + nullable: true + type: string + repo: + description: Repo is the name of the HTTPS helm repo to + download the chart from. + nullable: true + type: string + skipSchemaValidation: + description: SkipSchemaValidation allows skipping schema + validation against the chart values + type: boolean + takeOwnership: + description: TakeOwnership makes helm skip the check for + its own annotations + type: boolean + timeoutSeconds: + description: TimeoutSeconds is the time to wait for Helm + operations. + type: integer + values: + description: 'Values passed to Helm. It is possible to specify + the keys and values + + as go template strings.' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + description: ValuesFiles is a list of files to load values + from. + items: + nullable: true + type: string + nullable: true + type: array + valuesFrom: + description: ValuesFrom loads the values from configmaps + and secrets. + items: + description: 'Define helm values that can come from configmap, + secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439' + properties: + configMapKeyRef: + description: The reference to a config map with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + description: The reference to a secret with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + description: Version of the chart to download + nullable: true + type: string + waitForJobs: + description: 'WaitForJobs if set and timeoutSeconds provided, + will wait until all + + Jobs have been completed before marking the GitRepo as + ready. It + + will wait for as long as timeoutSeconds' + type: boolean + type: object + ignore: + description: IgnoreOptions can be used to ignore fields when + monitoring the bundle. + properties: + conditions: + description: Conditions is a list of conditions to be ignored + when monitoring the Bundle. + items: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + nullable: true + type: array + type: object + keepResources: + description: KeepResources can be used to keep the deployed + resources when removing the bundle + type: boolean + kustomize: + description: 'Kustomize options for the deployment, like the + dir containing the + + kustomization.yaml file.' + nullable: true + properties: + dir: + description: 'Dir points to a custom folder for kustomize + resources. This folder must contain + + a kustomization.yaml file.' + nullable: true + type: string + type: object + namespace: + description: 'TargetNamespace if present will assign all resource + to this + + namespace and if any cluster scoped resource exists the deployment + + will fail.' + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + nullable: true + type: string + description: NamespaceAnnotations are annotations that will + be appended to the namespace created by Fleet. + nullable: true + type: object + namespaceLabels: + additionalProperties: + nullable: true + type: string + description: NamespaceLabels are labels that will be appended + to the namespace created by Fleet. + nullable: true + type: object + serviceAccount: + description: ServiceAccount which will be used to perform this + deployment. + nullable: true + type: string + yaml: + description: 'YAML options, if using raw YAML these are names + that map to + + overlays/{name} files that will be used to replace or patch + a resource.' + nullable: true + properties: + overlays: + description: 'Overlays is a list of names that maps to folders + in "overlays/". + + If you wish to customize the file ./subdir/resource.yaml + then a file + + ./overlays/myoverlay/subdir/resource.yaml will replace + the base + + file. + + A file named ./overlays/myoverlay/subdir/resource_patch.yaml + will patch the base file.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + type: object + status: + properties: + appliedDeploymentID: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + nullable: true + type: string + lastUpdateTime: + description: The last time this condition was updated. + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + nullable: true + type: string + type: + description: Type of cluster condition. + nullable: true + type: string + type: object + nullable: true + type: array + display: + properties: + deployed: + nullable: true + type: string + monitored: + nullable: true + type: string + state: + nullable: true + type: string + type: object + modifiedStatus: + items: + description: 'ModifiedStatus is used to report the status of a + resource that is modified. + + It indicates if the modification was a create, a delete or a + patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + nonModified: + type: boolean + nonReadyStatus: + items: + description: NonReadyStatus is used to report the status of a + resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID values, including + UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names do not get + conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + ready: + type: boolean + release: + nullable: true + type: string + resources: + description: 'Resources lists the metadata of resources that were + deployed + + according to the helm release history.' + items: + description: BundleDeploymentResource contains the metadata of + a deployed resource. + properties: + apiVersion: + nullable: true + type: string + createdAt: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + nullable: true + type: array + syncGeneration: + nullable: true + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bundlenamespacemappings.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: BundleNamespaceMapping + plural: bundlenamespacemappings + singular: bundlenamespacemapping + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: BundleNamespaceMapping maps bundles to clusters in other namespaces. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + bundleSelector: + description: 'A label selector is a label query over a set of resources. + The result of matchLabels and + + matchExpressions are ANDed. An empty label selector matches all objects. + A null + + label selector matches no objects.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: 'A label selector requirement is a selector that + contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector applies + to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship to + a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. If the + operator is In or NotIn, + + the values array must be non-empty. If the operator is Exists + or DoesNotExist, + + the values array must be empty. This array is replaced during + a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, whose key + field is "key", the + + operator is "In", and the values array contains only "value". + The requirements are ANDed.' + nullable: true + type: object + type: object + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + namespaceSelector: + description: 'A label selector is a label query over a set of resources. + The result of matchLabels and + + matchExpressions are ANDed. An empty label selector matches all objects. + A null + + label selector matches no objects.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: 'A label selector requirement is a selector that + contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector applies + to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship to + a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. If the + operator is In or NotIn, + + the values array must be non-empty. If the operator is Exists + or DoesNotExist, + + the values array must be empty. This array is replaced during + a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, whose key + field is "key", the + + operator is "In", and the values array contains only "value". + The requirements are ANDed.' + nullable: true + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bundles.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: Bundle + plural: bundles + singular: bundle + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.readyClusters + name: BundleDeployments-Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'Bundle contains the resources of an application and its deployment + options. + + It will be deployed as a Helm chart to target clusters. + + + + When a GitRepo is scanned it will produce one or more bundles. Bundles + are + + a collection of resources that get deployed to one or more cluster(s). + Bundle is the + + fundamental deployment unit used in Fleet. The contents of a Bundle may + be + + Kubernetes manifests, Kustomize configuration, or Helm charts. Regardless + + of the source the contents are dynamically rendered into a Helm chart + by + + the agent and installed into the downstream cluster as a Helm release.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + nullable: true + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will be + used if true. This will try to recreate all resources in the + release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + defaultNamespace: + description: 'DefaultNamespace is the namespace to use for resources + that do not + + specify a namespace. This field is not used to enforce or lock + down + + the deployment to a specific namespace.' + nullable: true + type: string + deleteCRDResources: + description: DeleteCRDResources deletes CRDs. Warning! this will + also delete all your Custom Resources. + type: boolean + dependsOn: + description: DependsOn refers to the bundles which must be ready + before this bundle can be deployed. + items: + properties: + name: + description: Name of the bundle. + nullable: true + type: string + selector: + description: Selector matching bundle's labels. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + type: object + nullable: true + type: array + diff: + description: Diff can be used to ignore the modified state of objects + which are amended at runtime. + nullable: true + properties: + comparePatches: + description: ComparePatches match a resource and remove fields + from the check for modifications. + items: + description: ComparePatch matches a resource and removes fields + from the check for modifications. + properties: + apiVersion: + description: APIVersion is the apiVersion of the resource + to match. + nullable: true + type: string + jsonPointers: + description: JSONPointers ignore diffs at a certain JSON + path. + items: + nullable: true + type: string + nullable: true + type: array + kind: + description: Kind is the kind of the resource to match. + nullable: true + type: string + name: + description: Name is the name of the resource to match. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource + to match. + nullable: true + type: string + operations: + description: Operations remove a JSON path from the resource. + items: + description: Operation of a ComparePatch, usually "remove". + properties: + op: + description: Op is usually "remove" + nullable: true + type: string + path: + description: Path is the JSON path to remove. + nullable: true + type: string + value: + description: Value is usually empty. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + forceSyncGeneration: + description: ForceSyncGeneration is used to force a redeployment + type: integer + helm: + description: Helm options for the deployment, like the chart name, + repo and values. + nullable: true + properties: + atomic: + description: Atomic sets the --atomic flag when Helm is performing + an upgrade + type: boolean + chart: + description: 'Chart can refer to any go-getter URL or OCI registry + based helm + + chart URL. The chart will be downloaded.' + nullable: true + type: string + disableDNS: + description: DisableDNS can be used to customize Helm's EnableDNS + option, which Fleet sets to `true` by default. + type: boolean + disablePreProcess: + description: DisablePreProcess disables template processing + in values + type: boolean + force: + description: Force allows to override immutable resources. This + could be dangerous. + type: boolean + maxHistory: + description: MaxHistory limits the maximum number of revisions + saved per release by Helm. + type: integer + releaseName: + description: 'ReleaseName sets a custom release name to deploy + the chart as. If + + not specified a release name will be generated by combining + the + + invoking GitRepo.name + GitRepo.path.' + maxLength: 53 + nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + repo: + description: Repo is the name of the HTTPS helm repo to download + the chart from. + nullable: true + type: string + skipSchemaValidation: + description: SkipSchemaValidation allows skipping schema validation + against the chart values + type: boolean + takeOwnership: + description: TakeOwnership makes helm skip the check for its + own annotations + type: boolean + timeoutSeconds: + description: TimeoutSeconds is the time to wait for Helm operations. + type: integer + values: + description: 'Values passed to Helm. It is possible to specify + the keys and values + + as go template strings.' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + description: ValuesFiles is a list of files to load values from. + items: + nullable: true + type: string + nullable: true + type: array + valuesFrom: + description: ValuesFrom loads the values from configmaps and + secrets. + items: + description: 'Define helm values that can come from configmap, + secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439' + properties: + configMapKeyRef: + description: The reference to a config map with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + description: The reference to a secret with release values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + description: Version of the chart to download + nullable: true + type: string + waitForJobs: + description: 'WaitForJobs if set and timeoutSeconds provided, + will wait until all + + Jobs have been completed before marking the GitRepo as ready. + It + + will wait for as long as timeoutSeconds' + type: boolean + type: object + ignore: + description: IgnoreOptions can be used to ignore fields when monitoring + the bundle. + properties: + conditions: + description: Conditions is a list of conditions to be ignored + when monitoring the Bundle. + items: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + nullable: true + type: array + type: object + keepResources: + description: KeepResources can be used to keep the deployed resources + when removing the bundle + type: boolean + kustomize: + description: 'Kustomize options for the deployment, like the dir + containing the + + kustomization.yaml file.' + nullable: true + properties: + dir: + description: 'Dir points to a custom folder for kustomize resources. + This folder must contain + + a kustomization.yaml file.' + nullable: true + type: string + type: object + namespace: + description: 'TargetNamespace if present will assign all resource + to this + + namespace and if any cluster scoped resource exists the deployment + + will fail.' + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + nullable: true + type: string + description: NamespaceAnnotations are annotations that will be appended + to the namespace created by Fleet. + nullable: true + type: object + namespaceLabels: + additionalProperties: + nullable: true + type: string + description: NamespaceLabels are labels that will be appended to + the namespace created by Fleet. + nullable: true + type: object + paused: + description: Paused if set to true, will stop any BundleDeployments + from being updated. It will be marked as out of sync. + type: boolean + resources: + description: 'Resources contains the resources that were read from + the bundle''s + + path. This includes the content of downloaded helm charts.' + items: + description: BundleResource represents the content of a single + resource from the bundle, like a YAML manifest. + properties: + content: + description: The content of the resource, can be compressed. + nullable: true + type: string + encoding: + description: Encoding is either empty or "base64+gz". + nullable: true + type: string + name: + description: Name of the resource, can include the bundle's + internal path. + nullable: true + type: string + type: object + nullable: true + type: array + rolloutStrategy: + description: 'RolloutStrategy controls the rollout of bundles, by + defining + + partitions, canaries and percentages for cluster availability.' + nullable: true + properties: + autoPartitionSize: + description: 'A number or percentage of how to automatically + partition clusters if no + + specific partitioning strategy is configured. + + default: 25%' + nullable: true + x-kubernetes-int-or-string: true + maxUnavailable: + description: 'A number or percentage of clusters that can be + unavailable during an update + + of a bundle. This follows the same basic approach as a deployment + rollout + + strategy. Once the number of clusters meets unavailable state + update will be + + paused. Default value is 100% which doesn''t take effect on + update. + + default: 100%' + nullable: true + x-kubernetes-int-or-string: true + maxUnavailablePartitions: + description: 'A number or percentage of cluster partitions that + can be unavailable during + + an update of a bundle. + + default: 0' + nullable: true + x-kubernetes-int-or-string: true + partitions: + description: 'A list of definitions of partitions. If any target + clusters do not match + + the configuration they are added to partitions at the end + following the + + autoPartitionSize.' + items: + description: Partition defines a separate rollout strategy + for a set of clusters. + properties: + clusterGroup: + description: A cluster group name to include in this partition + nullable: true + type: string + clusterGroupSelector: + description: Selector matching cluster group labels to + include in this partition + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a + selector that contains values, a key, and an operator + that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and + DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the + operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + clusterName: + description: ClusterName is the name of a cluster to include + in this partition + nullable: true + type: string + clusterSelector: + description: Selector matching cluster labels to include + in this partition + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a + selector that contains values, a key, and an operator + that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and + DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the + operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + maxUnavailable: + description: 'A number or percentage of clusters that + can be unavailable in this + + partition before this partition is treated as done. + + default: 10%' + nullable: true + x-kubernetes-int-or-string: true + name: + description: A user-friendly name given to the partition + used for Display (optional). + nullable: true + type: string + type: object + nullable: true + type: array + type: object + serviceAccount: + description: ServiceAccount which will be used to perform this deployment. + nullable: true + type: string + targetRestrictions: + description: TargetRestrictions is an allow list, which controls + if a bundledeployment is created for a target. + items: + description: 'BundleTargetRestriction is used internally by Fleet + and should not be modified. + + It acts as an allow list, to prevent the creation of BundleDeployments + from + + Targets created by TargetCustomizations in fleet.yaml.' + properties: + clusterGroup: + nullable: true + type: string + clusterGroupSelector: + description: 'A label selector is a label query over a set + of resources. The result of matchLabels and + + matchExpressions are ANDed. An empty label selector matches + all objects. A null + + label selector matches no objects.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + clusterName: + nullable: true + type: string + clusterSelector: + description: 'A label selector is a label query over a set + of resources. The result of matchLabels and + + matchExpressions are ANDed. An empty label selector matches + all objects. A null + + label selector matches no objects.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + name: + nullable: true + type: string + type: object + nullable: true + type: array + targets: + description: 'Targets refer to the clusters which will be deployed + to. + + Targets are evaluated in order and the first one to match is used.' + items: + description: 'BundleTarget declares clusters to deploy to. Fleet + will merge the + + BundleDeploymentOptions from customizations into this struct.' + properties: + clusterGroup: + description: ClusterGroup to match a specific cluster group + by name. + nullable: true + type: string + clusterGroupSelector: + description: ClusterGroupSelector is a selector to match cluster + groups. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + clusterName: + description: 'ClusterName to match a specific cluster by name + that will be + + selected' + nullable: true + type: string + clusterSelector: + description: 'ClusterSelector is a selector to match clusters. + The structure is + + the standard metav1.LabelSelector format. If clusterGroupSelector + or + + clusterGroup is specified, clusterSelector will be used + only to + + further refine the selection after clusterGroupSelector + and + + clusterGroup is evaluated.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + nullable: true + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will + be used if true. This will try to recreate all resources + in the release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + defaultNamespace: + description: 'DefaultNamespace is the namespace to use for + resources that do not + + specify a namespace. This field is not used to enforce or + lock down + + the deployment to a specific namespace.' + nullable: true + type: string + deleteCRDResources: + description: DeleteCRDResources deletes CRDs. Warning! this + will also delete all your Custom Resources. + type: boolean + diff: + description: Diff can be used to ignore the modified state + of objects which are amended at runtime. + nullable: true + properties: + comparePatches: + description: ComparePatches match a resource and remove + fields from the check for modifications. + items: + description: ComparePatch matches a resource and removes + fields from the check for modifications. + properties: + apiVersion: + description: APIVersion is the apiVersion of the + resource to match. + nullable: true + type: string + jsonPointers: + description: JSONPointers ignore diffs at a certain + JSON path. + items: + nullable: true + type: string + nullable: true + type: array + kind: + description: Kind is the kind of the resource to + match. + nullable: true + type: string + name: + description: Name is the name of the resource to + match. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource + to match. + nullable: true + type: string + operations: + description: Operations remove a JSON path from + the resource. + items: + description: Operation of a ComparePatch, usually + "remove". + properties: + op: + description: Op is usually "remove" + nullable: true + type: string + path: + description: Path is the JSON path to remove. + nullable: true + type: string + value: + description: Value is usually empty. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + doNotDeploy: + description: DoNotDeploy if set to true, will not deploy to + this target. + type: boolean + forceSyncGeneration: + description: ForceSyncGeneration is used to force a redeployment + type: integer + helm: + description: Helm options for the deployment, like the chart + name, repo and values. + nullable: true + properties: + atomic: + description: Atomic sets the --atomic flag when Helm is + performing an upgrade + type: boolean + chart: + description: 'Chart can refer to any go-getter URL or + OCI registry based helm + + chart URL. The chart will be downloaded.' + nullable: true + type: string + disableDNS: + description: DisableDNS can be used to customize Helm's + EnableDNS option, which Fleet sets to `true` by default. + type: boolean + disablePreProcess: + description: DisablePreProcess disables template processing + in values + type: boolean + force: + description: Force allows to override immutable resources. + This could be dangerous. + type: boolean + maxHistory: + description: MaxHistory limits the maximum number of revisions + saved per release by Helm. + type: integer + releaseName: + description: 'ReleaseName sets a custom release name to + deploy the chart as. If + + not specified a release name will be generated by combining + the + + invoking GitRepo.name + GitRepo.path.' + nullable: true + type: string + repo: + description: Repo is the name of the HTTPS helm repo to + download the chart from. + nullable: true + type: string + skipSchemaValidation: + description: SkipSchemaValidation allows skipping schema + validation against the chart values + type: boolean + takeOwnership: + description: TakeOwnership makes helm skip the check for + its own annotations + type: boolean + timeoutSeconds: + description: TimeoutSeconds is the time to wait for Helm + operations. + type: integer + values: + description: 'Values passed to Helm. It is possible to + specify the keys and values + + as go template strings.' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + description: ValuesFiles is a list of files to load values + from. + items: + nullable: true + type: string + nullable: true + type: array + valuesFrom: + description: ValuesFrom loads the values from configmaps + and secrets. + items: + description: 'Define helm values that can come from + configmap, secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439' + properties: + configMapKeyRef: + description: The reference to a config map with + release values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same + namespace as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + description: The reference to a secret with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same + namespace as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + description: Version of the chart to download + nullable: true + type: string + waitForJobs: + description: 'WaitForJobs if set and timeoutSeconds provided, + will wait until all + + Jobs have been completed before marking the GitRepo + as ready. It + + will wait for as long as timeoutSeconds' + type: boolean + type: object + ignore: + description: IgnoreOptions can be used to ignore fields when + monitoring the bundle. + properties: + conditions: + description: Conditions is a list of conditions to be + ignored when monitoring the Bundle. + items: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + nullable: true + type: array + type: object + keepResources: + description: KeepResources can be used to keep the deployed + resources when removing the bundle + type: boolean + kustomize: + description: 'Kustomize options for the deployment, like the + dir containing the + + kustomization.yaml file.' + nullable: true + properties: + dir: + description: 'Dir points to a custom folder for kustomize + resources. This folder must contain + + a kustomization.yaml file.' + nullable: true + type: string + type: object + name: + description: 'Name of target. This value is largely for display + and logging. If + + not specified a default name of the format "target000" will + be used' + nullable: true + type: string + namespace: + description: 'TargetNamespace if present will assign all resource + to this + + namespace and if any cluster scoped resource exists the + deployment + + will fail.' + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + nullable: true + type: string + description: NamespaceAnnotations are annotations that will + be appended to the namespace created by Fleet. + nullable: true + type: object + namespaceLabels: + additionalProperties: + nullable: true + type: string + description: NamespaceLabels are labels that will be appended + to the namespace created by Fleet. + nullable: true + type: object + serviceAccount: + description: ServiceAccount which will be used to perform + this deployment. + nullable: true + type: string + yaml: + description: 'YAML options, if using raw YAML these are names + that map to + + overlays/{name} files that will be used to replace or patch + a resource.' + nullable: true + properties: + overlays: + description: 'Overlays is a list of names that maps to + folders in "overlays/". + + If you wish to customize the file ./subdir/resource.yaml + then a file + + ./overlays/myoverlay/subdir/resource.yaml will replace + the base + + file. + + A file named ./overlays/myoverlay/subdir/resource_patch.yaml + will patch the base file.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + nullable: true + type: array + yaml: + description: 'YAML options, if using raw YAML these are names that + map to + + overlays/{name} files that will be used to replace or patch a + resource.' + nullable: true + properties: + overlays: + description: 'Overlays is a list of names that maps to folders + in "overlays/". + + If you wish to customize the file ./subdir/resource.yaml then + a file + + ./overlays/myoverlay/subdir/resource.yaml will replace the + base + + file. + + A file named ./overlays/myoverlay/subdir/resource_patch.yaml + will patch the base file.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + status: + properties: + conditions: + description: 'Conditions is a list of Wrangler conditions that describe + the state + + of the bundle.' + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + nullable: true + type: string + lastUpdateTime: + description: The last time this condition was updated. + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + nullable: true + type: string + type: + description: Type of cluster condition. + nullable: true + type: string + type: object + nullable: true + type: array + display: + description: 'Display contains the number of ready, desiredready + clusters and a + + summary state for the bundle''s resources.' + properties: + readyClusters: + description: 'ReadyClusters is a string in the form "%d/%d", + that describes the + + number of clusters that are ready vs. the number of clusters + desired + + to be ready.' + nullable: true + type: string + state: + description: State is a summary state for the bundle, calculated + over the non-ready resources. + nullable: true + type: string + type: object + maxNew: + description: 'MaxNew is always 50. A bundle change can only stage + 50 + + bundledeployments at a time.' + type: integer + maxUnavailable: + description: 'MaxUnavailable is the maximum number of unavailable + deployments. See + + rollout configuration.' + type: integer + maxUnavailablePartitions: + description: 'MaxUnavailablePartitions is the maximum number of + unavailable + + partitions. The rollout configuration defines a maximum number + or + + percentage of unavailable partitions.' + type: integer + newlyCreated: + description: 'NewlyCreated is the number of bundle deployments that + have been created, + + not updated.' + type: integer + observedGeneration: + description: ObservedGeneration is the current generation of the + bundle. + type: integer + partitions: + description: PartitionStatus lists the status of each partition. + items: + description: PartitionStatus is the status of a single rollout + partition. + properties: + count: + description: Count is the number of clusters in the partition. + type: integer + maxUnavailable: + description: MaxUnavailable is the maximum number of unavailable + clusters in the partition. + type: integer + name: + description: Name is the name of the partition. + nullable: true + type: string + summary: + description: Summary is a summary state for the partition, + calculated over its non-ready resources. + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, + but with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments + that have been deployed + + and for which all resources are ready, but where some + changes from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which + is filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information + about a bundle that is not ready for a + + given state like "ErrApplied". It contains a list + of non-ready or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, + like e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the + bundle is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for + each modified resource. + items: + description: 'ModifiedStatus is used to report + the status of a resource that is modified. + + It indicates if the modification was a create, + a delete or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for + each non-ready resource. + items: + description: NonReadyStatus is used to report + the status of a resource that is not ready. + It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique + ID values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias + to string. Being a type captures + + intent and helps make sure that UIDs and + names do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments + that have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream + agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments + that are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments + that have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, + but are waiting + + to be deployed.' + type: integer + type: object + unavailable: + description: Unavailable is the number of unavailable clusters + in the partition. + type: integer + type: object + nullable: true + type: array + resourceKey: + description: 'ResourceKey lists resources, which will likely be + deployed. The + + actual list of resources on a cluster might differ, depending + on the + + helm chart, value templating, etc..' + items: + description: ResourceKey lists resources, which will likely be + deployed. + properties: + apiVersion: + description: APIVersion is the k8s api version of the resource. + nullable: true + type: string + kind: + description: Kind is the k8s api kind of the resource. + nullable: true + type: string + name: + description: Name is the name of the resource. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource. + nullable: true + type: string + type: object + nullable: true + type: array + summary: + description: 'Summary contains the number of bundle deployments + in each state and + + a list of non-ready resources.' + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, but + with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments that + have been deployed + + and for which all resources are ready, but where some changes + from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which is + filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information about + a bundle that is not ready for a + + given state like "ErrApplied". It contains a list of non-ready + or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, like + e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the bundle + is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for each modified + resource. + items: + description: 'ModifiedStatus is used to report the status + of a resource that is modified. + + It indicates if the modification was a create, a delete + or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for each non-ready + resource. + items: + description: NonReadyStatus is used to report the status + of a resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID + values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names + do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments that + have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments that + are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments that + have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, but are + waiting + + to be deployed.' + type: integer + type: object + unavailable: + description: 'Unavailable is the number of bundle deployments that + are not ready or + + where the AppliedDeploymentID in the status does not match the + + DeploymentID from the spec.' + type: integer + unavailablePartitions: + description: UnavailablePartitions is the number of unavailable + partitions. + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clustergroups.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + categories: + - fleet + kind: ClusterGroup + plural: clustergroups + singular: clustergroup + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.readyClusters + name: Clusters-Ready + type: string + - jsonPath: .status.display.readyBundles + name: Bundles-Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterGroup is a re-usable selector to target a group of clusters. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + selector: + description: Selector is a label selector, used to select clusters + for this group. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: 'A label selector requirement is a selector that + contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector applies + to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. If + the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is replaced + during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, whose + key field is "key", the + + operator is "In", and the values array contains only "value". + The requirements are ANDed.' + nullable: true + type: object + type: object + type: object + status: + properties: + clusterCount: + description: ClusterCount is the number of clusters in the cluster + group. + type: integer + conditions: + description: Conditions is a list of conditions and their statuses + for the cluster group. + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + nullable: true + type: string + lastUpdateTime: + description: The last time this condition was updated. + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + nullable: true + type: string + type: + description: Type of cluster condition. + nullable: true + type: string + type: object + nullable: true + type: array + display: + description: 'Display contains the number of ready, desiredready + clusters and a + + summary state for the bundle''s resources.' + properties: + readyBundles: + description: 'ReadyBundles is a string in the form "%d/%d", + that describes the + + number of bundles that are ready vs. the number of bundles + desired + + to be ready.' + nullable: true + type: string + readyClusters: + description: 'ReadyClusters is a string in the form "%d/%d", + that describes the + + number of clusters that are ready vs. the number of clusters + desired + + to be ready.' + nullable: true + type: string + state: + description: 'State is a summary state for the cluster group, + showing "NotReady" if + + there are non-ready resources.' + nullable: true + type: string + type: object + nonReadyClusterCount: + description: NonReadyClusterCount is the number of clusters that + are not ready. + type: integer + nonReadyClusters: + description: NonReadyClusters is a list of cluster names that are + not ready. + items: + nullable: true + type: string + nullable: true + type: array + resourceCounts: + description: 'ResourceCounts contains the number of resources in + each state over + + all bundles in the cluster group.' + properties: + desiredReady: + description: DesiredReady is the number of resources that should + be ready. + type: integer + missing: + description: Missing is the number of missing resources. + type: integer + modified: + description: Modified is the number of resources that have been + modified. + type: integer + notReady: + description: 'NotReady is the number of not ready resources. + Resources are not + + ready if they do not match any other state.' + type: integer + orphaned: + description: Orphaned is the number of orphaned resources. + type: integer + ready: + description: Ready is the number of ready resources. + type: integer + unknown: + description: Unknown is the number of resources in an unknown + state. + type: integer + waitApplied: + description: WaitApplied is the number of resources that are + waiting to be applied. + type: integer + type: object + summary: + description: 'Summary is a summary of the bundle deployments and + their resources + + in the cluster group.' + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, but + with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments that + have been deployed + + and for which all resources are ready, but where some changes + from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which is + filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information about + a bundle that is not ready for a + + given state like "ErrApplied". It contains a list of non-ready + or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, like + e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the bundle + is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for each modified + resource. + items: + description: 'ModifiedStatus is used to report the status + of a resource that is modified. + + It indicates if the modification was a create, a delete + or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for each non-ready + resource. + items: + description: NonReadyStatus is used to report the status + of a resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID + values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names + do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments that + have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments that + are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments that + have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, but are + waiting + + to be deployed.' + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterregistrations.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: ClusterRegistration + plural: clusterregistrations + singular: clusterregistration + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.clusterName + name: Cluster-Name + type: string + - jsonPath: .spec.clusterLabels + name: Labels + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterRegistration is used internally by Fleet and should + not be used directly. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + clientID: + description: 'ClientID is a unique string that will identify the + cluster. The + + agent either uses the configured ID or the kubeSystem.UID.' + nullable: true + type: string + clientRandom: + description: 'ClientRandom is a random string that the agent generates. + When + + fleet-controller grants a registration, it creates a registration + + secret with this string in the name.' + nullable: true + type: string + clusterLabels: + additionalProperties: + nullable: true + type: string + description: ClusterLabels are copied to the cluster resource during + the registration. + nullable: true + type: object + type: object + status: + properties: + clusterName: + description: 'ClusterName is only set after the registration is + being processed by + + fleet-controller.' + nullable: true + type: string + granted: + description: 'Granted is set to true, if the request service account + is present + + and its token secret exists. This happens directly before creating + + the registration secret, roles and rolebindings.' + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterregistrationtokens.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: ClusterRegistrationToken + plural: clusterregistrationtokens + singular: clusterregistrationtoken + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.secretName + name: Secret-Name + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterRegistrationToken is used by agents to register a new + cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + properties: + name: + maxLength: 63 + pattern: ^[-a-z0-9]+$ + type: string + type: object + spec: + properties: + ttl: + description: 'TTL is the time to live for the token. It is used + to calculate the + + expiration time. If the token expires, it will be deleted.' + nullable: true + type: string + type: object + status: + properties: + expires: + description: Expires is the time when the token expires. + nullable: true + type: string + secretName: + description: SecretName is the name of the secret containing the + token. + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusters.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: Cluster + plural: clusters + singular: cluster + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.readyBundles + name: Bundles-Ready + type: string + - jsonPath: .status.display.readyNodes + name: Nodes-Ready + type: string + - jsonPath: .status.display.sampleNode + name: Sample-Node + type: string + - jsonPath: .status.agent.lastSeen + name: Last-Seen + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'Cluster corresponds to a Kubernetes cluster. Fleet deploys + bundles to targeted clusters. + + Clusters to which Fleet deploys manifests are referred to as downstream + + clusters. In the single cluster use case, the Fleet manager Kubernetes + + cluster is both the manager and downstream cluster at the same time.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + properties: + name: + maxLength: 63 + pattern: ^[-a-z0-9]+$ + type: string + type: object + spec: + properties: + agentAffinity: + description: 'AgentAffinity overrides the default affinity for the + cluster''s agent + + deployment. If this value is nil the default affinity is used.' + nullable: true + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the + pod. + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: 'The scheduler will prefer to schedule pods + to nodes that satisfy + + the affinity expressions specified by this field, but + it may choose + + a node that violates one or more of the expressions. The + node that is + + most preferred is the one with the greatest sum of weights, + i.e. + + for each node that meets all of the scheduling requirements + (resource + + request, requiredDuringScheduling affinity expressions, + etc.), + + compute a sum by iterating through the elements of this + field and adding + + "weight" to the sum if the node matches the corresponding + matchExpressions; the + + node(s) with the highest sum are the most preferred.' + items: + description: 'An empty preferred scheduling term matches + all objects with implicit weight 0 + + (i.e. it''s a no-op). A null preferred scheduling term + matches no objects (i.e. is also a no-op).' + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: 'A node selector requirement is + a selector that contains values, a key, and + an operator + + that relates the key and values.' + properties: + key: + description: The label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'Represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + nullable: true + type: string + values: + description: 'An array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. If the + operator is Gt or Lt, the values + + array must have a single element, which + will be interpreted as an integer. + + This array is replaced during a strategic + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: 'A node selector requirement is + a selector that contains values, a key, and + an operator + + that relates the key and values.' + properties: + key: + description: The label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'Represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + nullable: true + type: string + values: + description: 'An array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. If the + operator is Gt or Lt, the values + + array must have a single element, which + will be interpreted as an integer. + + This array is replaced during a strategic + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: 'If the affinity requirements specified by + this field are not met at + + scheduling time, the pod will not be scheduled onto the + node. + + If the affinity requirements specified by this field cease + to be met + + at some point during pod execution (e.g. due to an update), + the system + + may or may not try to eventually evict the pod from its + node.' + nullable: true + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: 'A null or empty node selector term matches + no objects. The requirements of + + them are ANDed. + + The TopologySelectorTerm type implements a subset + of the NodeSelectorTerm.' + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: 'A node selector requirement is + a selector that contains values, a key, and + an operator + + that relates the key and values.' + properties: + key: + description: The label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'Represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + nullable: true + type: string + values: + description: 'An array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. If the + operator is Gt or Lt, the values + + array must have a single element, which + will be interpreted as an integer. + + This array is replaced during a strategic + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: 'A node selector requirement is + a selector that contains values, a key, and + an operator + + that relates the key and values.' + properties: + key: + description: The label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'Represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + nullable: true + type: string + values: + description: 'An array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. If the + operator is Gt or Lt, the values + + array must have a single element, which + will be interpreted as an integer. + + This array is replaced during a strategic + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: 'The scheduler will prefer to schedule pods + to nodes that satisfy + + the affinity expressions specified by this field, but + it may choose + + a node that violates one or more of the expressions. The + node that is + + most preferred is the one with the greatest sum of weights, + i.e. + + for each node that meets all of the scheduling requirements + (resource + + request, requiredDuringScheduling affinity expressions, + etc.), + + compute a sum by iterating through the elements of this + field and adding + + "weight" to the sum if the node has pods which matches + the corresponding podAffinityTerm; the + + node(s) with the highest sum are the most preferred.' + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: 'A label selector requirement + is a selector that contains values, a + key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that + the selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a + key''s relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of + string values. If the operator is + In or NotIn, + + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + + the values array must be empty. This + array is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaceSelector: + description: 'A label query over the set of namespaces + that the term applies to. + + The term is applied to the union of the namespaces + selected by this field + + and the ones listed in the namespaces field. + + null selector and null or empty namespaces list + means "this pod''s namespace". + + An empty selector ({}) matches all namespaces.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: 'A label selector requirement + is a selector that contains values, a + key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that + the selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a + key''s relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of + string values. If the operator is + In or NotIn, + + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + + the values array must be empty. This + array is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaces: + description: 'namespaces specifies a static list + of namespace names that the term applies to. + + The term is applied to the union of the namespaces + listed in this field + + and the ones selected by namespaceSelector. + + null or empty namespaces list and null namespaceSelector + means "this pod''s namespace".' + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + description: 'This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching + + the labelSelector in the specified namespaces, + where co-located is defined as running on a + node + + whose value of the label with key topologyKey + matches that of any node on which any of the + + selected pods is running. + + Empty topologyKey is not allowed.' + nullable: true + type: string + type: object + weight: + description: 'weight associated with matching the + corresponding podAffinityTerm, + + in the range 1-100.' + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: 'If the affinity requirements specified by + this field are not met at + + scheduling time, the pod will not be scheduled onto the + node. + + If the affinity requirements specified by this field cease + to be met + + at some point during pod execution (e.g. due to a pod + label update), the + + system may or may not try to eventually evict the pod + from its node. + + When there are multiple elements, the lists of nodes corresponding + to each + + podAffinityTerm are intersected, i.e. all terms must be + satisfied.' + items: + description: 'Defines a set of pods (namely those matching + the labelSelector + + relative to the given namespace(s)) that this pod should + be + + co-located (affinity) or not co-located (anti-affinity) + with, + + where co-located is defined as running on a node whose + value of + + the label with key matches that of any + node on which + + a pod of the set of pods is running' + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: 'A label selector requirement is + a selector that contains values, a key, and + an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the + selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s + relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of string + values. If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaceSelector: + description: 'A label query over the set of namespaces + that the term applies to. + + The term is applied to the union of the namespaces + selected by this field + + and the ones listed in the namespaces field. + + null selector and null or empty namespaces list + means "this pod''s namespace". + + An empty selector ({}) matches all namespaces.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: 'A label selector requirement is + a selector that contains values, a key, and + an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the + selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s + relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of string + values. If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaces: + description: 'namespaces specifies a static list of + namespace names that the term applies to. + + The term is applied to the union of the namespaces + listed in this field + + and the ones selected by namespaceSelector. + + null or empty namespaces list and null namespaceSelector + means "this pod''s namespace".' + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + description: 'This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching + + the labelSelector in the specified namespaces, where + co-located is defined as running on a node + + whose value of the label with key topologyKey matches + that of any node on which any of the + + selected pods is running. + + Empty topologyKey is not allowed.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: 'The scheduler will prefer to schedule pods + to nodes that satisfy + + the anti-affinity expressions specified by this field, + but it may choose + + a node that violates one or more of the expressions. The + node that is + + most preferred is the one with the greatest sum of weights, + i.e. + + for each node that meets all of the scheduling requirements + (resource + + request, requiredDuringScheduling anti-affinity expressions, + etc.), + + compute a sum by iterating through the elements of this + field and adding + + "weight" to the sum if the node has pods which matches + the corresponding podAffinityTerm; the + + node(s) with the highest sum are the most preferred.' + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: 'A label selector requirement + is a selector that contains values, a + key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that + the selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a + key''s relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of + string values. If the operator is + In or NotIn, + + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + + the values array must be empty. This + array is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaceSelector: + description: 'A label query over the set of namespaces + that the term applies to. + + The term is applied to the union of the namespaces + selected by this field + + and the ones listed in the namespaces field. + + null selector and null or empty namespaces list + means "this pod''s namespace". + + An empty selector ({}) matches all namespaces.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: 'A label selector requirement + is a selector that contains values, a + key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that + the selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a + key''s relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of + string values. If the operator is + In or NotIn, + + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + + the values array must be empty. This + array is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaces: + description: 'namespaces specifies a static list + of namespace names that the term applies to. + + The term is applied to the union of the namespaces + listed in this field + + and the ones selected by namespaceSelector. + + null or empty namespaces list and null namespaceSelector + means "this pod''s namespace".' + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + description: 'This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching + + the labelSelector in the specified namespaces, + where co-located is defined as running on a + node + + whose value of the label with key topologyKey + matches that of any node on which any of the + + selected pods is running. + + Empty topologyKey is not allowed.' + nullable: true + type: string + type: object + weight: + description: 'weight associated with matching the + corresponding podAffinityTerm, + + in the range 1-100.' + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: 'If the anti-affinity requirements specified + by this field are not met at + + scheduling time, the pod will not be scheduled onto the + node. + + If the anti-affinity requirements specified by this field + cease to be met + + at some point during pod execution (e.g. due to a pod + label update), the + + system may or may not try to eventually evict the pod + from its node. + + When there are multiple elements, the lists of nodes corresponding + to each + + podAffinityTerm are intersected, i.e. all terms must be + satisfied.' + items: + description: 'Defines a set of pods (namely those matching + the labelSelector + + relative to the given namespace(s)) that this pod should + be + + co-located (affinity) or not co-located (anti-affinity) + with, + + where co-located is defined as running on a node whose + value of + + the label with key matches that of any + node on which + + a pod of the set of pods is running' + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: 'A label selector requirement is + a selector that contains values, a key, and + an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the + selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s + relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of string + values. If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaceSelector: + description: 'A label query over the set of namespaces + that the term applies to. + + The term is applied to the union of the namespaces + selected by this field + + and the ones listed in the namespaces field. + + null selector and null or empty namespaces list + means "this pod''s namespace". + + An empty selector ({}) matches all namespaces.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: 'A label selector requirement is + a selector that contains values, a key, and + an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the + selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s + relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of string + values. If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaces: + description: 'namespaces specifies a static list of + namespace names that the term applies to. + + The term is applied to the union of the namespaces + listed in this field + + and the ones selected by namespaceSelector. + + null or empty namespaces list and null namespaceSelector + means "this pod''s namespace".' + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + description: 'This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching + + the labelSelector in the specified namespaces, where + co-located is defined as running on a node + + whose value of the label with key topologyKey matches + that of any node on which any of the + + selected pods is running. + + Empty topologyKey is not allowed.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + type: object + agentEnvVars: + description: AgentEnvVars are extra environment variables to be + added to the agent deployment. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + nullable: true + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + + using the previously defined environment variables in the + container and + + any service environment variables. If a variable cannot + be resolved, + + the reference in the input string will be unchanged. Double + $$ are reduced + + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. + + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + + Escaped references will never be expanded, regardless of + whether the variable + + exists or not. + + Defaults to "".' + nullable: true + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + nullable: true + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + nullable: true + properties: + key: + description: The key to select. + nullable: true + type: string + name: + description: 'Name of the referent. + + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + + TODO: Add other useful fields. apiVersion, kind, + uid?' + nullable: true + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + nullable: true + type: boolean + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, `metadata.annotations['''']`, + + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP, status.podIPs.' + nullable: true + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + nullable: true + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + nullable: true + type: string + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests + + (limits.cpu, limits.memory, limits.ephemeral-storage, + requests.cpu, requests.memory and requests.ephemeral-storage) + are currently supported.' + nullable: true + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + nullable: true + type: string + divisor: + description: Specifies the output format of the exposed + resources, defaults to "1" + nullable: true + type: string + resource: + description: 'Required: resource to select' + nullable: true + type: string + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + nullable: true + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + nullable: true + type: string + name: + description: 'Name of the referent. + + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + + TODO: Add other useful fields. apiVersion, kind, + uid?' + nullable: true + type: string + optional: + description: Specify whether the Secret or its key + must be defined + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + agentNamespace: + description: AgentNamespace defaults to the system namespace, e.g. + cattle-fleet-system. + nullable: true + type: string + agentResources: + description: AgentResources sets the resources for the cluster's + agent deployment. + nullable: true + properties: + claims: + description: 'Claims lists the names of resources, defined in + spec.resourceClaims, + + that are used by this container. + + + + This is an alpha field and requires enabling the + + DynamicResourceAllocation feature gate. + + + + This field is immutable. It can only be set for containers.' + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: 'Name must match the name of one entry in + pod.spec.resourceClaims of + + the Pod where this field is used. It makes that resource + available + + inside a container.' + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + description: 'Limits describes the maximum amount of compute + resources allowed. + + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + description: 'Requests describes the minimum amount of compute + resources required. + + If Requests is omitted for a container, it defaults to Limits + if that is explicitly specified, + + otherwise to an implementation-defined value. Requests cannot + exceed Limits. + + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + nullable: true + type: object + type: object + agentTolerations: + description: AgentTolerations defines an extra set of Tolerations + to be added to the Agent deployment. + items: + description: 'The pod this Toleration is attached to tolerates + any taint that matches + + the triple using the matching operator .' + properties: + effect: + description: 'Effect indicates the taint effect to match. + Empty means match all taint effects. + + When specified, allowed values are NoSchedule, PreferNoSchedule + and NoExecute.' + nullable: true + type: string + key: + description: 'Key is the taint key that the toleration applies + to. Empty means match all taint keys. + + If the key is empty, operator must be Exists; this combination + means to match all values and all keys.' + nullable: true + type: string + operator: + description: 'Operator represents a key''s relationship to + the value. + + Valid operators are Exists and Equal. Defaults to Equal. + + Exists is equivalent to wildcard for value, so that a pod + can + + tolerate all taints of a particular category.' + nullable: true + type: string + tolerationSeconds: + description: 'TolerationSeconds represents the period of time + the toleration (which must be + + of effect NoExecute, otherwise this field is ignored) tolerates + the taint. By default, + + it is not set, which means tolerate the taint forever (do + not evict). Zero and + + negative values will be treated as 0 (evict immediately) + by the system.' + maximum: 86400 + nullable: true + type: integer + value: + description: 'Value is the taint value the toleration matches + to. + + If the operator is Exists, the value should be empty, otherwise + just a regular string.' + nullable: true + type: string + type: object + nullable: true + type: array + clientID: + description: 'ClientID is a unique string that will identify the + cluster. It can + + either be predefined, or generated when importing the cluster.' + nullable: true + type: string + kubeConfigSecret: + description: 'KubeConfigSecret is the name of the secret containing + the kubeconfig for the downstream cluster. + + It can optionally contain a APIServerURL and CA to override the + + values in the fleet-controller''s configmap.' + nullable: true + type: string + kubeConfigSecretNamespace: + description: 'KubeConfigSecretNamespace is the namespace of the + secret containing the kubeconfig for the downstream cluster. + + If unset, it will be assumed the secret can be found in the namespace + that the Cluster object resides within.' + nullable: true + type: string + paused: + description: Paused if set to true, will stop any BundleDeployments + from being updated. + type: boolean + privateRepoURL: + description: PrivateRepoURL prefixes the image name and overrides + a global repo URL from the agents config. + nullable: true + type: string + redeployAgentGeneration: + description: RedeployAgentGeneration can be used to force redeploying + the agent. + type: integer + templateValues: + description: TemplateValues defines a cluster specific mapping of + values to be sent to fleet.yaml values templating. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + status: + properties: + agent: + description: AgentStatus contains information about the agent. + properties: + lastSeen: + description: 'LastSeen is the last time the agent checked in + to update the status + + of the cluster resource.' + nullable: true + type: string + namespace: + description: Namespace is the namespace of the agent deployment, + e.g. "cattle-fleet-system". + nullable: true + type: string + nonReadyNodeNames: + description: 'NonReadyNode contains the names of non-ready nodes. + The list is + + limited to at most 3 names.' + items: + nullable: true + type: string + nullable: true + type: array + nonReadyNodes: + description: NonReadyNodes is the number of nodes that are not + ready. + type: integer + readyNodeNames: + description: 'ReadyNodes contains the names of ready nodes. + The list is limited to + + at most 3 names.' + items: + nullable: true + type: string + nullable: true + type: array + readyNodes: + description: ReadyNodes is the number of nodes that are ready. + type: integer + type: object + agentAffinityHash: + description: 'AgentAffinityHash is a hash of the agent''s affinity + configuration, + + used to detect changes.' + nullable: true + type: string + agentConfigChanged: + description: 'AgentConfigChanged is set to true if any of the agent + configuration + + changed, like the API server URL or CA. Setting it to true will + + trigger a re-import of the cluster.' + type: boolean + agentDeployedGeneration: + description: AgentDeployedGeneration is the generation of the agent + that is currently deployed. + nullable: true + type: integer + agentEnvVarsHash: + description: AgentEnvVarsHash is a hash of the agent's env vars, + used to detect changes. + nullable: true + type: string + agentMigrated: + description: 'AgentMigrated is always set to true after importing + a cluster. If + + false, it will trigger a migration. Old agents don''t have + + this in their status.' + type: boolean + agentNamespaceMigrated: + description: 'AgentNamespaceMigrated is always set to true after + importing a + + cluster. If false, it will trigger a migration. Old Fleet agents + + don''t have this in their status.' + type: boolean + agentPrivateRepoURL: + description: AgentPrivateRepoURL is the private repo URL for the + agent that is currently used. + nullable: true + type: string + agentResourcesHash: + description: 'AgentResourcesHash is a hash of the agent''s resources + configuration, + + used to detect changes.' + nullable: true + type: string + agentTLSMode: + description: 'AgentTLSMode supports two values: `system-store` and + `strict`. If set to + + `system-store`, instructs the agent to trust CA bundles from the + operating + + system''s store. If set to `strict`, then the agent shall only + connect to a + + server which uses the exact CA configured when creating/updating + the agent.' + nullable: true + type: string + agentTolerationsHash: + description: 'AgentTolerationsHash is a hash of the agent''s tolerations + + configuration, used to detect changes.' + nullable: true + type: string + apiServerCAHash: + description: APIServerCAHash is a hash of the upstream API server + CA, used to detect changes. + nullable: true + type: string + apiServerURL: + description: 'APIServerURL is the currently used URL of the API + server that the + + cluster uses to connect to upstream.' + nullable: true + type: string + cattleNamespaceMigrated: + description: 'CattleNamespaceMigrated is always set to true after + importing a + + cluster. If false, it will trigger a migration. Old Fleet agents, + + don''t have this in their status.' + type: boolean + conditions: + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + nullable: true + type: string + lastUpdateTime: + description: The last time this condition was updated. + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + nullable: true + type: string + type: + description: Type of cluster condition. + nullable: true + type: string + type: object + nullable: true + type: array + desiredReadyGitRepos: + description: 'DesiredReadyGitRepos is the number of gitrepos for + this cluster that + + are desired to be ready.' + type: integer + display: + description: Display contains the number of ready bundles, nodes + and a summary state. + properties: + readyBundles: + description: 'ReadyBundles is a string in the form "%d/%d", + that describes the + + number of bundles that are ready vs. the number of bundles + desired + + to be ready.' + nullable: true + type: string + readyNodes: + description: 'ReadyNodes is a string in the form "%d/%d", that + describes the + + number of nodes that are ready vs. the number of expected + nodes.' + nullable: true + type: string + sampleNode: + description: 'SampleNode is the name of one of the nodes that + are ready. If no + + node is ready, it''s the name of a node that is not ready.' + nullable: true + type: string + state: + description: State of the cluster, either one of the bundle + states, or "WaitCheckIn". + nullable: true + type: string + type: object + namespace: + description: 'Namespace is the cluster namespace, it contains the + clusters service + + account as well as any bundledeployments. Example: + + "cluster-fleet-local-cluster-294db1acfa77-d9ccf852678f"' + nullable: true + type: string + readyGitRepos: + description: ReadyGitRepos is the number of gitrepos for this cluster + that are ready. + type: integer + resourceCounts: + description: ResourceCounts is an aggregate over the GitRepoResourceCounts. + properties: + desiredReady: + description: DesiredReady is the number of resources that should + be ready. + type: integer + missing: + description: Missing is the number of missing resources. + type: integer + modified: + description: Modified is the number of resources that have been + modified. + type: integer + notReady: + description: 'NotReady is the number of not ready resources. + Resources are not + + ready if they do not match any other state.' + type: integer + orphaned: + description: Orphaned is the number of orphaned resources. + type: integer + ready: + description: Ready is the number of ready resources. + type: integer + unknown: + description: Unknown is the number of resources in an unknown + state. + type: integer + waitApplied: + description: WaitApplied is the number of resources that are + waiting to be applied. + type: integer + type: object + summary: + description: 'Summary is a summary of the bundledeployments. The + resource counts + + are copied from the gitrepo resource.' + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, but + with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments that + have been deployed + + and for which all resources are ready, but where some changes + from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which is + filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information about + a bundle that is not ready for a + + given state like "ErrApplied". It contains a list of non-ready + or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, like + e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the bundle + is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for each modified + resource. + items: + description: 'ModifiedStatus is used to report the status + of a resource that is modified. + + It indicates if the modification was a create, a delete + or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for each non-ready + resource. + items: + description: NonReadyStatus is used to report the status + of a resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID + values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names + do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments that + have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments that + are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments that + have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, but are + waiting + + to be deployed.' + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: contents.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: Content + plural: contents + singular: content + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: 'Content is used internally by Fleet and should not be used + directly. It + + contains the resources from a bundle for a specific target cluster.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + content: + description: 'Content is a byte array, which contains the manifests + of a bundle. + + The bundle resources are copied into the bundledeployment''s content + + resource, so the downstream agent can deploy them.' + nullable: true + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gitreporestrictions.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: GitRepoRestriction + plural: gitreporestrictions + singular: gitreporestriction + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .defaultServiceAccount + name: Default-ServiceAccount + type: string + - jsonPath: .allowedServiceAccounts + name: Allowed-ServiceAccounts + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'GitRepoRestriction is a resource that can optionally be used + to restrict + + the options of GitRepos in the same namespace.' + properties: + allowedClientSecretNames: + description: AllowedClientSecretNames is a list of client secret names + that GitRepos are allowed to use. + items: + nullable: true + type: string + nullable: true + type: array + allowedRepoPatterns: + description: 'AllowedRepoPatterns is a list of regex patterns that restrict + the + + valid values of the Repo field of a GitRepo.' + items: + nullable: true + type: string + nullable: true + type: array + allowedServiceAccounts: + description: AllowedServiceAccounts is a list of service accounts that + GitRepos are allowed to use. + items: + nullable: true + type: string + nullable: true + type: array + allowedTargetNamespaces: + description: 'AllowedTargetNamespaces restricts TargetNamespace to the + given + + namespaces. If AllowedTargetNamespaces is set, TargetNamespace must + + be set.' + items: + nullable: true + type: string + nullable: true + type: array + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + defaultClientSecretName: + description: DefaultClientSecretName overrides the GitRepo's default + client secret. + nullable: true + type: string + defaultServiceAccount: + description: DefaultServiceAccount overrides the GitRepo's default service + account. + nullable: true + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gitrepos.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + categories: + - fleet + kind: GitRepo + plural: gitrepos + singular: gitrepo + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.repo + name: Repo + type: string + - jsonPath: .status.commit + name: Commit + type: string + - jsonPath: .status.display.readyBundleDeployments + name: BundleDeployments-Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'GitRepo describes a git repository that is watched by Fleet. + + The resource contains the necessary information to deploy the repo, or + parts + + of it, to target clusters.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + branch: + description: Branch The git branch to follow. + nullable: true + type: string + caBundle: + description: CABundle is a PEM encoded CA bundle which will be used + to validate the repo's certificate. + nullable: true + type: string + clientSecretName: + description: 'ClientSecretName is the name of the client secret + to be used to connect to the repo + + It is expected the secret be of type "kubernetes.io/basic-auth" + or "kubernetes.io/ssh-auth".' + nullable: true + type: string + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + nullable: true + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will be + used if true. This will try to recreate all resources in the + release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + forceSyncGeneration: + description: Increment this number to force a redeployment of contents + from Git. + type: integer + helmRepoURLRegex: + description: 'HelmRepoURLRegex Helm credentials will be used if + the helm repo matches this regex + + Credentials will always be used if this is empty or not provided.' + nullable: true + type: string + helmSecretName: + description: HelmSecretName contains the auth secret for a private + Helm repository. + nullable: true + type: string + helmSecretNameForPaths: + description: HelmSecretNameForPaths contains the auth secret for + private Helm repository for each path. + nullable: true + type: string + imageScanCommit: + description: Commit specifies how to commit to the git repo when + a new image is scanned and written back to git repo. + properties: + authorEmail: + description: AuthorEmail gives the email to provide when making + a commit + nullable: true + type: string + authorName: + description: AuthorName gives the name to provide when making + a commit + nullable: true + type: string + messageTemplate: + description: 'MessageTemplate provides a template for the commit + message, + + into which will be interpolated the details of the change + made.' + nullable: true + type: string + type: object + imageScanInterval: + description: ImageScanInterval is the interval of syncing scanned + images and writing back to git repo. + nullable: true + type: string + insecureSkipTLSVerify: + description: InsecureSkipTLSverify will use insecure HTTPS to clone + the repo. + type: boolean + keepResources: + description: KeepResources specifies if the resources created must + be kept after deleting the GitRepo. + type: boolean + paths: + description: 'Paths is the directories relative to the git repo + root that contain resources to be applied. + + Path globbing is supported, for example ["charts/*"] will match + all folders as a subdirectory of charts/ + + If empty, "/" is the default.' + items: + nullable: true + type: string + nullable: true + type: array + paused: + description: 'Paused, when true, causes changes in Git not to be + propagated down to the clusters but instead to mark + + resources as OutOfSync.' + type: boolean + pollingInterval: + description: PollingInterval is how often to check git for new updates. + nullable: true + type: string + repo: + description: Repo is a URL to a git repo to clone and index. + nullable: true + type: string + revision: + description: Revision A specific commit or tag to operate on. + nullable: true + type: string + serviceAccount: + description: ServiceAccount used in the downstream cluster for deployment. + nullable: true + type: string + targetNamespace: + description: 'Ensure that all resources are created in this namespace + + Any cluster scoped resource will be rejected if this is set + + Additionally this namespace will be created on demand.' + nullable: true + type: string + targets: + description: Targets is a list of targets this repo will deploy + to. + items: + description: GitTarget is a cluster or cluster group to deploy + to. + properties: + clusterGroup: + description: ClusterGroup is the name of a cluster group in + the same namespace as the clusters. + nullable: true + type: string + clusterGroupSelector: + description: ClusterGroupSelector is a label selector to select + cluster groups. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + clusterName: + description: ClusterName is the name of a cluster. + nullable: true + type: string + clusterSelector: + description: ClusterSelector is a label selector to select + clusters. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + name: + description: Name is the name of this target. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + status: + properties: + commit: + description: Commit is the Git commit hash from the last gitjob + run. + nullable: true + type: string + conditions: + description: 'Conditions is a list of Wrangler conditions that describe + the state + + of the GitRepo.' + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + nullable: true + type: string + lastUpdateTime: + description: The last time this condition was updated. + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + nullable: true + type: string + type: + description: Type of cluster condition. + nullable: true + type: string + type: object + nullable: true + type: array + desiredReadyClusters: + description: "DesiredReadyClusters\tis the number of clusters that\ + \ should be ready for bundles of this GitRepo." + type: integer + display: + description: Display contains a human readable summary of the status. + properties: + error: + description: Error is true if a message is present. + type: boolean + message: + description: Message contains the relevant message from the + deployment conditions. + nullable: true + type: string + readyBundleDeployments: + description: 'ReadyBundleDeployments is a string in the form + "%d/%d", that describes the + + number of ready bundledeployments over the total number of + bundledeployments.' + nullable: true + type: string + state: + description: 'State is the state of the GitRepo, e.g. "GitUpdating" + or the maximal + + BundleState according to StateRank.' + nullable: true + type: string + type: object + gitJobStatus: + description: GitJobStatus is the status of the last GitJob run, + e.g. "Current" if there was no error. + nullable: true + type: string + lastSyncedImageScanTime: + description: LastSyncedImageScanTime is the time of the last image + scan. + nullable: true + type: string + observedGeneration: + description: 'ObservedGeneration is the current generation of the + resource in the cluster. It is copied from k8s + + metadata.Generation. The value is incremented for all changes, + except for changes to .metadata or .status.' + type: integer + readyClusters: + description: 'ReadyClusters is the lowest number of clusters that + are ready over + + all the bundles of this GitRepo.' + type: integer + resourceCounts: + description: ResourceCounts contains the number of resources in + each state over all bundles. + properties: + desiredReady: + description: DesiredReady is the number of resources that should + be ready. + type: integer + missing: + description: Missing is the number of missing resources. + type: integer + modified: + description: Modified is the number of resources that have been + modified. + type: integer + notReady: + description: 'NotReady is the number of not ready resources. + Resources are not + + ready if they do not match any other state.' + type: integer + orphaned: + description: Orphaned is the number of orphaned resources. + type: integer + ready: + description: Ready is the number of ready resources. + type: integer + unknown: + description: Unknown is the number of resources in an unknown + state. + type: integer + waitApplied: + description: WaitApplied is the number of resources that are + waiting to be applied. + type: integer + type: object + resourceErrors: + description: ResourceErrors is a sorted list of errors from the + resources. + items: + nullable: true + type: string + nullable: true + type: array + resources: + description: Resources contains metadata about the resources of + each bundle. + items: + description: GitRepoResource contains metadata about the resources + of a bundle. + properties: + apiVersion: + description: APIVersion is the API version of the resource. + nullable: true + type: string + error: + description: Error is true if any Error in the PerClusterState + is true. + type: boolean + id: + description: ID is the name of the resource, e.g. "namespace1/my-config" + or "backingimagemanagers.storage.io". + nullable: true + type: string + incompleteState: + description: 'IncompleteState is true if a bundle summary + has 10 or more non-ready + + resources or a non-ready resource has more 10 or more non-ready + or + + modified states.' + type: boolean + kind: + description: Kind is the k8s kind of the resource. + nullable: true + type: string + message: + description: Message is the first message from the PerClusterStates. + nullable: true + type: string + name: + description: Name of the resource. + nullable: true + type: string + namespace: + description: Namespace of the resource. + nullable: true + type: string + perClusterState: + description: PerClusterState is a list of states for each + cluster. Derived from the summaries non-ready resources. + items: + description: ResourcePerClusterState is generated for each + non-ready resource of the bundles. + properties: + clusterId: + description: ClusterID is the id of the cluster. + nullable: true + type: string + error: + description: Error is true if the resource is in an + error state, copied from the bundle's summary for + non-ready resources. + type: boolean + message: + description: Message combines the messages from the + bundle's summary. Messages are joined with the delimiter + ';'. + nullable: true + type: string + patch: + description: Patch for modified resources. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + state: + description: State is the state of the resource. + nullable: true + type: string + transitioning: + description: 'Transitioning is true if the resource + is in a transitioning state, + + copied from the bundle''s summary for non-ready resources.' + type: boolean + type: object + nullable: true + type: array + state: + description: State is the state of the resource, e.g. "Unknown", + "WaitApplied", "ErrApplied" or "Ready". + nullable: true + type: string + transitioning: + description: Transitioning is true if any Transitioning in + the PerClusterState is true. + type: boolean + type: + description: Type is the type of the resource, e.g. "apiextensions.k8s.io.customresourcedefinition" + or "configmap". + nullable: true + type: string + type: object + nullable: true + type: array + summary: + description: Summary contains the number of bundle deployments in + each state and a list of non-ready resources. + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, but + with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments that + have been deployed + + and for which all resources are ready, but where some changes + from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which is + filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information about + a bundle that is not ready for a + + given state like "ErrApplied". It contains a list of non-ready + or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, like + e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the bundle + is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for each modified + resource. + items: + description: 'ModifiedStatus is used to report the status + of a resource that is modified. + + It indicates if the modification was a create, a delete + or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for each non-ready + resource. + items: + description: NonReadyStatus is used to report the status + of a resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID + values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names + do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments that + have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments that + are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments that + have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, but are + waiting + + to be deployed.' + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: imagescans.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + categories: + - fleet + kind: ImageScan + plural: imagescans + singular: imagescan + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.image + name: Repository + type: string + - jsonPath: .status.latestTag + name: Latest + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: API is taken from https://github.com/fluxcd/image-reflector-controller + properties: + gitrepoName: + description: GitRepo reference name + nullable: true + type: string + image: + description: Image is the name of the image repository + nullable: true + type: string + interval: + description: 'Interval is the length of time to wait between + + scans of the image repository.' + nullable: true + type: string + policy: + description: 'Policy gives the particulars of the policy to be followed + in + + selecting the most recent image' + properties: + alphabetical: + description: Alphabetical set of rules to use for alphabetical + ordering of the tags. + nullable: true + properties: + order: + description: 'Order specifies the sorting order of the tags. + Given the letters of the + + alphabet as tags, ascending order would select Z, and + descending order + + would select A.' + nullable: true + type: string + type: object + semver: + description: 'SemVer gives a semantic version range to check + against the tags + + available.' + nullable: true + properties: + range: + description: 'Range gives a semver range for the image tag; + the highest + + version within the range that''s a tag yields the latest + image.' + nullable: true + type: string + type: object + type: object + secretRef: + description: 'SecretRef can be given the name of a secret containing + + credentials to use for the image registry. The secret should be + + created with `kubectl create secret docker-registry`, or the + + equivalent.' + nullable: true + properties: + name: + description: 'Name of the referent. + + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + + TODO: Add other useful fields. apiVersion, kind, uid?' + nullable: true + type: string + type: object + suspend: + description: 'This flag tells the controller to suspend subsequent + image scans. + + It does not apply to already started scans. Defaults to false.' + type: boolean + tagName: + description: TagName is the tag ref that needs to be put in manifest + to replace fields + nullable: true + type: string + type: object + status: + properties: + canonicalImageName: + description: 'CanonicalName is the name of the image repository + with all the + + implied bits made explicit; e.g., `docker.io/library/alpine` + + rather than `alpine`.' + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + nullable: true + type: string + lastUpdateTime: + description: The last time this condition was updated. + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + nullable: true + type: string + type: + description: Type of cluster condition. + nullable: true + type: string + type: object + nullable: true + type: array + lastScanTime: + description: LastScanTime is the last time image was scanned + nullable: true + type: string + latestDigest: + description: LatestDigest is the digest of latest tag + nullable: true + type: string + latestImage: + description: 'LatestImage gives the first in the list of images + scanned by + + the image repository, when filtered and ordered according to + + the policy.' + nullable: true + type: string + latestTag: + description: Latest tag is the latest tag filtered by the policy + nullable: true + type: string + observedGeneration: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/fleet-crd/103.1.13+up0.9.15/templates/gitjobs-crds.yaml b/charts/fleet-crd/103.1.13+up0.9.15/templates/gitjobs-crds.yaml new file mode 100644 index 0000000000..b5296dbaf5 --- /dev/null +++ b/charts/fleet-crd/103.1.13+up0.9.15/templates/gitjobs-crds.yaml @@ -0,0 +1,7690 @@ +{{- if .Capabilities.APIVersions.Has "apiextensions.k8s.io/v1" -}} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gitjobs.gitjob.cattle.io +spec: + group: gitjob.cattle.io + names: + kind: GitJob + plural: gitjobs + singular: gitjob + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.git.repo + name: REPO + type: string + - jsonPath: .spec.git.branch + name: BRANCH + type: string + - jsonPath: .status.commit + name: COMMIT + type: string + - jsonPath: .status.jobStatus + name: JOBSTATUS + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + forceUpdateGeneration: + type: integer + git: + properties: + branch: + nullable: true + type: string + caBundle: + nullable: true + type: string + clientSecretName: + nullable: true + type: string + insecureSkipTLSVerify: + type: boolean + onTag: + nullable: true + type: string + provider: + nullable: true + type: string + repo: + nullable: true + type: string + revision: + nullable: true + type: string + type: object + jobSpec: + properties: + activeDeadlineSeconds: + nullable: true + type: integer + backoffLimit: + nullable: true + type: integer + backoffLimitPerIndex: + nullable: true + type: integer + completionMode: + nullable: true + type: string + completions: + nullable: true + type: integer + manualSelector: + nullable: true + type: boolean + maxFailedIndexes: + nullable: true + type: integer + parallelism: + nullable: true + type: integer + podFailurePolicy: + nullable: true + properties: + rules: + items: + properties: + action: + nullable: true + type: string + onExitCodes: + nullable: true + properties: + containerName: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + type: integer + nullable: true + type: array + type: object + onPodConditions: + items: + properties: + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + podReplacementPolicy: + nullable: true + type: string + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + suspend: + nullable: true + type: boolean + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + creationTimestamp: + nullable: true + type: string + deletionGracePeriodSeconds: + nullable: true + type: integer + deletionTimestamp: + nullable: true + type: string + finalizers: + items: + nullable: true + type: string + nullable: true + type: array + generateName: + nullable: true + type: string + generation: + type: integer + labels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + managedFields: + items: + properties: + apiVersion: + nullable: true + type: string + fieldsType: + nullable: true + type: string + fieldsV1: + nullable: true + type: object + manager: + nullable: true + type: string + operation: + nullable: true + type: string + subresource: + nullable: true + type: string + time: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + namespace: + nullable: true + type: string + ownerReferences: + items: + properties: + apiVersion: + nullable: true + type: string + blockOwnerDeletion: + nullable: true + type: boolean + controller: + nullable: true + type: boolean + kind: + nullable: true + type: string + name: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + nullable: true + type: array + resourceVersion: + nullable: true + type: string + selfLink: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + spec: + properties: + activeDeadlineSeconds: + nullable: true + type: integer + affinity: + nullable: true + properties: + nodeAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + nullable: true + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + type: object + podAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + podAntiAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + type: object + automountServiceAccountToken: + nullable: true + type: boolean + containers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + restartPolicy: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + dnsConfig: + nullable: true + properties: + nameservers: + items: + nullable: true + type: string + nullable: true + type: array + options: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + searches: + items: + nullable: true + type: string + nullable: true + type: array + type: object + dnsPolicy: + nullable: true + type: string + enableServiceLinks: + nullable: true + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + restartPolicy: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + nullable: true + type: string + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + hostAliases: + items: + properties: + hostnames: + items: + nullable: true + type: string + nullable: true + type: array + ip: + nullable: true + type: string + type: object + nullable: true + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostUsers: + nullable: true + type: boolean + hostname: + nullable: true + type: string + imagePullSecrets: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + initContainers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + restartPolicy: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + nodeName: + nullable: true + type: string + nodeSelector: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + os: + nullable: true + properties: + name: + nullable: true + type: string + type: object + overhead: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + preemptionPolicy: + nullable: true + type: string + priority: + nullable: true + type: integer + priorityClassName: + nullable: true + type: string + readinessGates: + items: + properties: + conditionType: + nullable: true + type: string + type: object + nullable: true + type: array + resourceClaims: + items: + properties: + name: + nullable: true + type: string + source: + properties: + resourceClaimName: + nullable: true + type: string + resourceClaimTemplateName: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + restartPolicy: + nullable: true + type: string + runtimeClassName: + nullable: true + type: string + schedulerName: + nullable: true + type: string + schedulingGates: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + securityContext: + nullable: true + properties: + fsGroup: + nullable: true + type: integer + fsGroupChangePolicy: + nullable: true + type: string + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + supplementalGroups: + items: + type: integer + nullable: true + type: array + sysctls: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + serviceAccount: + nullable: true + type: string + serviceAccountName: + nullable: true + type: string + setHostnameAsFQDN: + nullable: true + type: boolean + shareProcessNamespace: + nullable: true + type: boolean + subdomain: + nullable: true + type: string + terminationGracePeriodSeconds: + nullable: true + type: integer + tolerations: + items: + properties: + effect: + nullable: true + type: string + key: + nullable: true + type: string + operator: + nullable: true + type: string + tolerationSeconds: + nullable: true + type: integer + value: + nullable: true + type: string + type: object + nullable: true + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + matchLabelKeys: + items: + nullable: true + type: string + nullable: true + type: array + maxSkew: + type: integer + minDomains: + nullable: true + type: integer + nodeAffinityPolicy: + nullable: true + type: string + nodeTaintsPolicy: + nullable: true + type: string + topologyKey: + nullable: true + type: string + whenUnsatisfiable: + nullable: true + type: string + type: object + nullable: true + type: array + volumes: + items: + properties: + awsElasticBlockStore: + nullable: true + properties: + fsType: + nullable: true + type: string + partition: + type: integer + readOnly: + type: boolean + volumeID: + nullable: true + type: string + type: object + azureDisk: + nullable: true + properties: + cachingMode: + nullable: true + type: string + diskName: + nullable: true + type: string + diskURI: + nullable: true + type: string + fsType: + nullable: true + type: string + kind: + nullable: true + type: string + readOnly: + nullable: true + type: boolean + type: object + azureFile: + nullable: true + properties: + readOnly: + type: boolean + secretName: + nullable: true + type: string + shareName: + nullable: true + type: string + type: object + cephfs: + nullable: true + properties: + monitors: + items: + nullable: true + type: string + nullable: true + type: array + path: + nullable: true + type: string + readOnly: + type: boolean + secretFile: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + user: + nullable: true + type: string + type: object + cinder: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + volumeID: + nullable: true + type: string + type: object + configMap: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + csi: + nullable: true + properties: + driver: + nullable: true + type: string + fsType: + nullable: true + type: string + nodePublishSecretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + readOnly: + nullable: true + type: boolean + volumeAttributes: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + downwardAPI: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + mode: + nullable: true + type: integer + path: + nullable: true + type: string + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + emptyDir: + nullable: true + properties: + medium: + nullable: true + type: string + sizeLimit: + nullable: true + type: string + type: object + ephemeral: + nullable: true + properties: + volumeClaimTemplate: + nullable: true + properties: + metadata: + properties: + annotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + creationTimestamp: + nullable: true + type: string + deletionGracePeriodSeconds: + nullable: true + type: integer + deletionTimestamp: + nullable: true + type: string + finalizers: + items: + nullable: true + type: string + nullable: true + type: array + generateName: + nullable: true + type: string + generation: + type: integer + labels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + managedFields: + items: + properties: + apiVersion: + nullable: true + type: string + fieldsType: + nullable: true + type: string + fieldsV1: + nullable: true + type: object + manager: + nullable: true + type: string + operation: + nullable: true + type: string + subresource: + nullable: true + type: string + time: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + namespace: + nullable: true + type: string + ownerReferences: + items: + properties: + apiVersion: + nullable: true + type: string + blockOwnerDeletion: + nullable: true + type: boolean + controller: + nullable: true + type: boolean + kind: + nullable: true + type: string + name: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + nullable: true + type: array + resourceVersion: + nullable: true + type: string + selfLink: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + spec: + properties: + accessModes: + items: + nullable: true + type: string + nullable: true + type: array + dataSource: + nullable: true + properties: + apiGroup: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + type: object + dataSourceRef: + nullable: true + properties: + apiGroup: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + storageClassName: + nullable: true + type: string + volumeMode: + nullable: true + type: string + volumeName: + nullable: true + type: string + type: object + type: object + type: object + fc: + nullable: true + properties: + fsType: + nullable: true + type: string + lun: + nullable: true + type: integer + readOnly: + type: boolean + targetWWNs: + items: + nullable: true + type: string + nullable: true + type: array + wwids: + items: + nullable: true + type: string + nullable: true + type: array + type: object + flexVolume: + nullable: true + properties: + driver: + nullable: true + type: string + fsType: + nullable: true + type: string + options: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + type: object + flocker: + nullable: true + properties: + datasetName: + nullable: true + type: string + datasetUUID: + nullable: true + type: string + type: object + gcePersistentDisk: + nullable: true + properties: + fsType: + nullable: true + type: string + partition: + type: integer + pdName: + nullable: true + type: string + readOnly: + type: boolean + type: object + gitRepo: + nullable: true + properties: + directory: + nullable: true + type: string + repository: + nullable: true + type: string + revision: + nullable: true + type: string + type: object + glusterfs: + nullable: true + properties: + endpoints: + nullable: true + type: string + path: + nullable: true + type: string + readOnly: + type: boolean + type: object + hostPath: + nullable: true + properties: + path: + nullable: true + type: string + type: + nullable: true + type: string + type: object + iscsi: + nullable: true + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + nullable: true + type: string + initiatorName: + nullable: true + type: string + iqn: + nullable: true + type: string + iscsiInterface: + nullable: true + type: string + lun: + type: integer + portals: + items: + nullable: true + type: string + nullable: true + type: array + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + targetPortal: + nullable: true + type: string + type: object + name: + nullable: true + type: string + nfs: + nullable: true + properties: + path: + nullable: true + type: string + readOnly: + type: boolean + server: + nullable: true + type: string + type: object + persistentVolumeClaim: + nullable: true + properties: + claimName: + nullable: true + type: string + readOnly: + type: boolean + type: object + photonPersistentDisk: + nullable: true + properties: + fsType: + nullable: true + type: string + pdID: + nullable: true + type: string + type: object + portworxVolume: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + volumeID: + nullable: true + type: string + type: object + projected: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + sources: + items: + properties: + configMap: + nullable: true + properties: + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + downwardAPI: + nullable: true + properties: + items: + items: + properties: + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + mode: + nullable: true + type: integer + path: + nullable: true + type: string + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + secret: + nullable: true + properties: + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + serviceAccountToken: + nullable: true + properties: + audience: + nullable: true + type: string + expirationSeconds: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + quobyte: + nullable: true + properties: + group: + nullable: true + type: string + readOnly: + type: boolean + registry: + nullable: true + type: string + tenant: + nullable: true + type: string + user: + nullable: true + type: string + volume: + nullable: true + type: string + type: object + rbd: + nullable: true + properties: + fsType: + nullable: true + type: string + image: + nullable: true + type: string + keyring: + nullable: true + type: string + monitors: + items: + nullable: true + type: string + nullable: true + type: array + pool: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + user: + nullable: true + type: string + type: object + scaleIO: + nullable: true + properties: + fsType: + nullable: true + type: string + gateway: + nullable: true + type: string + protectionDomain: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + sslEnabled: + type: boolean + storageMode: + nullable: true + type: string + storagePool: + nullable: true + type: string + system: + nullable: true + type: string + volumeName: + nullable: true + type: string + type: object + secret: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + optional: + nullable: true + type: boolean + secretName: + nullable: true + type: string + type: object + storageos: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + volumeName: + nullable: true + type: string + volumeNamespace: + nullable: true + type: string + type: object + vsphereVolume: + nullable: true + properties: + fsType: + nullable: true + type: string + storagePolicyID: + nullable: true + type: string + storagePolicyName: + nullable: true + type: string + volumePath: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + type: object + ttlSecondsAfterFinished: + nullable: true + type: integer + type: object + syncInterval: + type: integer + type: object + status: + properties: + commit: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + event: + nullable: true + type: string + hookId: + nullable: true + type: string + jobStatus: + nullable: true + type: string + lastExecutedCommit: + nullable: true + type: string + lastSyncedTime: + nullable: true + type: string + observedGeneration: + type: integer + secretToken: + nullable: true + type: string + updateGeneration: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +{{- else -}} +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: gitjobs.gitjob.cattle.io +spec: + additionalPrinterColumns: + - JSONPath: .spec.git.repo + name: REPO + type: string + - JSONPath: .spec.git.branch + name: BRANCH + type: string + - JSONPath: .status.commit + name: COMMIT + type: string + - JSONPath: .status.jobStatus + name: JOBSTATUS + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: gitjob.cattle.io + names: + kind: GitJob + plural: gitjobs + singular: gitjob + preserveUnknownFields: false + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + forceUpdateGeneration: + type: integer + git: + properties: + branch: + nullable: true + type: string + caBundle: + nullable: true + type: string + clientSecretName: + nullable: true + type: string + insecureSkipTLSVerify: + type: boolean + onTag: + nullable: true + type: string + provider: + nullable: true + type: string + repo: + nullable: true + type: string + revision: + nullable: true + type: string + type: object + jobSpec: + properties: + activeDeadlineSeconds: + nullable: true + type: integer + backoffLimit: + nullable: true + type: integer + backoffLimitPerIndex: + nullable: true + type: integer + completionMode: + nullable: true + type: string + completions: + nullable: true + type: integer + manualSelector: + nullable: true + type: boolean + maxFailedIndexes: + nullable: true + type: integer + parallelism: + nullable: true + type: integer + podFailurePolicy: + nullable: true + properties: + rules: + items: + properties: + action: + nullable: true + type: string + onExitCodes: + nullable: true + properties: + containerName: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + type: integer + nullable: true + type: array + type: object + onPodConditions: + items: + properties: + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + podReplacementPolicy: + nullable: true + type: string + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + suspend: + nullable: true + type: boolean + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + creationTimestamp: + nullable: true + type: string + deletionGracePeriodSeconds: + nullable: true + type: integer + deletionTimestamp: + nullable: true + type: string + finalizers: + items: + nullable: true + type: string + nullable: true + type: array + generateName: + nullable: true + type: string + generation: + type: integer + labels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + managedFields: + items: + properties: + apiVersion: + nullable: true + type: string + fieldsType: + nullable: true + type: string + fieldsV1: + nullable: true + type: object + manager: + nullable: true + type: string + operation: + nullable: true + type: string + subresource: + nullable: true + type: string + time: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + namespace: + nullable: true + type: string + ownerReferences: + items: + properties: + apiVersion: + nullable: true + type: string + blockOwnerDeletion: + nullable: true + type: boolean + controller: + nullable: true + type: boolean + kind: + nullable: true + type: string + name: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + nullable: true + type: array + resourceVersion: + nullable: true + type: string + selfLink: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + spec: + properties: + activeDeadlineSeconds: + nullable: true + type: integer + affinity: + nullable: true + properties: + nodeAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + nullable: true + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + type: object + podAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + podAntiAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + type: object + automountServiceAccountToken: + nullable: true + type: boolean + containers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + restartPolicy: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + dnsConfig: + nullable: true + properties: + nameservers: + items: + nullable: true + type: string + nullable: true + type: array + options: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + searches: + items: + nullable: true + type: string + nullable: true + type: array + type: object + dnsPolicy: + nullable: true + type: string + enableServiceLinks: + nullable: true + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + restartPolicy: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + nullable: true + type: string + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + hostAliases: + items: + properties: + hostnames: + items: + nullable: true + type: string + nullable: true + type: array + ip: + nullable: true + type: string + type: object + nullable: true + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostUsers: + nullable: true + type: boolean + hostname: + nullable: true + type: string + imagePullSecrets: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + initContainers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + restartPolicy: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + nodeName: + nullable: true + type: string + nodeSelector: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + os: + nullable: true + properties: + name: + nullable: true + type: string + type: object + overhead: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + preemptionPolicy: + nullable: true + type: string + priority: + nullable: true + type: integer + priorityClassName: + nullable: true + type: string + readinessGates: + items: + properties: + conditionType: + nullable: true + type: string + type: object + nullable: true + type: array + resourceClaims: + items: + properties: + name: + nullable: true + type: string + source: + properties: + resourceClaimName: + nullable: true + type: string + resourceClaimTemplateName: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + restartPolicy: + nullable: true + type: string + runtimeClassName: + nullable: true + type: string + schedulerName: + nullable: true + type: string + schedulingGates: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + securityContext: + nullable: true + properties: + fsGroup: + nullable: true + type: integer + fsGroupChangePolicy: + nullable: true + type: string + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + supplementalGroups: + items: + type: integer + nullable: true + type: array + sysctls: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + serviceAccount: + nullable: true + type: string + serviceAccountName: + nullable: true + type: string + setHostnameAsFQDN: + nullable: true + type: boolean + shareProcessNamespace: + nullable: true + type: boolean + subdomain: + nullable: true + type: string + terminationGracePeriodSeconds: + nullable: true + type: integer + tolerations: + items: + properties: + effect: + nullable: true + type: string + key: + nullable: true + type: string + operator: + nullable: true + type: string + tolerationSeconds: + nullable: true + type: integer + value: + nullable: true + type: string + type: object + nullable: true + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + matchLabelKeys: + items: + nullable: true + type: string + nullable: true + type: array + maxSkew: + type: integer + minDomains: + nullable: true + type: integer + nodeAffinityPolicy: + nullable: true + type: string + nodeTaintsPolicy: + nullable: true + type: string + topologyKey: + nullable: true + type: string + whenUnsatisfiable: + nullable: true + type: string + type: object + nullable: true + type: array + volumes: + items: + properties: + awsElasticBlockStore: + nullable: true + properties: + fsType: + nullable: true + type: string + partition: + type: integer + readOnly: + type: boolean + volumeID: + nullable: true + type: string + type: object + azureDisk: + nullable: true + properties: + cachingMode: + nullable: true + type: string + diskName: + nullable: true + type: string + diskURI: + nullable: true + type: string + fsType: + nullable: true + type: string + kind: + nullable: true + type: string + readOnly: + nullable: true + type: boolean + type: object + azureFile: + nullable: true + properties: + readOnly: + type: boolean + secretName: + nullable: true + type: string + shareName: + nullable: true + type: string + type: object + cephfs: + nullable: true + properties: + monitors: + items: + nullable: true + type: string + nullable: true + type: array + path: + nullable: true + type: string + readOnly: + type: boolean + secretFile: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + user: + nullable: true + type: string + type: object + cinder: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + volumeID: + nullable: true + type: string + type: object + configMap: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + csi: + nullable: true + properties: + driver: + nullable: true + type: string + fsType: + nullable: true + type: string + nodePublishSecretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + readOnly: + nullable: true + type: boolean + volumeAttributes: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + downwardAPI: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + mode: + nullable: true + type: integer + path: + nullable: true + type: string + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + emptyDir: + nullable: true + properties: + medium: + nullable: true + type: string + sizeLimit: + nullable: true + type: string + type: object + ephemeral: + nullable: true + properties: + volumeClaimTemplate: + nullable: true + properties: + metadata: + properties: + annotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + creationTimestamp: + nullable: true + type: string + deletionGracePeriodSeconds: + nullable: true + type: integer + deletionTimestamp: + nullable: true + type: string + finalizers: + items: + nullable: true + type: string + nullable: true + type: array + generateName: + nullable: true + type: string + generation: + type: integer + labels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + managedFields: + items: + properties: + apiVersion: + nullable: true + type: string + fieldsType: + nullable: true + type: string + fieldsV1: + nullable: true + type: object + manager: + nullable: true + type: string + operation: + nullable: true + type: string + subresource: + nullable: true + type: string + time: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + namespace: + nullable: true + type: string + ownerReferences: + items: + properties: + apiVersion: + nullable: true + type: string + blockOwnerDeletion: + nullable: true + type: boolean + controller: + nullable: true + type: boolean + kind: + nullable: true + type: string + name: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + nullable: true + type: array + resourceVersion: + nullable: true + type: string + selfLink: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + spec: + properties: + accessModes: + items: + nullable: true + type: string + nullable: true + type: array + dataSource: + nullable: true + properties: + apiGroup: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + type: object + dataSourceRef: + nullable: true + properties: + apiGroup: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + storageClassName: + nullable: true + type: string + volumeMode: + nullable: true + type: string + volumeName: + nullable: true + type: string + type: object + type: object + type: object + fc: + nullable: true + properties: + fsType: + nullable: true + type: string + lun: + nullable: true + type: integer + readOnly: + type: boolean + targetWWNs: + items: + nullable: true + type: string + nullable: true + type: array + wwids: + items: + nullable: true + type: string + nullable: true + type: array + type: object + flexVolume: + nullable: true + properties: + driver: + nullable: true + type: string + fsType: + nullable: true + type: string + options: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + type: object + flocker: + nullable: true + properties: + datasetName: + nullable: true + type: string + datasetUUID: + nullable: true + type: string + type: object + gcePersistentDisk: + nullable: true + properties: + fsType: + nullable: true + type: string + partition: + type: integer + pdName: + nullable: true + type: string + readOnly: + type: boolean + type: object + gitRepo: + nullable: true + properties: + directory: + nullable: true + type: string + repository: + nullable: true + type: string + revision: + nullable: true + type: string + type: object + glusterfs: + nullable: true + properties: + endpoints: + nullable: true + type: string + path: + nullable: true + type: string + readOnly: + type: boolean + type: object + hostPath: + nullable: true + properties: + path: + nullable: true + type: string + type: + nullable: true + type: string + type: object + iscsi: + nullable: true + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + nullable: true + type: string + initiatorName: + nullable: true + type: string + iqn: + nullable: true + type: string + iscsiInterface: + nullable: true + type: string + lun: + type: integer + portals: + items: + nullable: true + type: string + nullable: true + type: array + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + targetPortal: + nullable: true + type: string + type: object + name: + nullable: true + type: string + nfs: + nullable: true + properties: + path: + nullable: true + type: string + readOnly: + type: boolean + server: + nullable: true + type: string + type: object + persistentVolumeClaim: + nullable: true + properties: + claimName: + nullable: true + type: string + readOnly: + type: boolean + type: object + photonPersistentDisk: + nullable: true + properties: + fsType: + nullable: true + type: string + pdID: + nullable: true + type: string + type: object + portworxVolume: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + volumeID: + nullable: true + type: string + type: object + projected: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + sources: + items: + properties: + configMap: + nullable: true + properties: + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + downwardAPI: + nullable: true + properties: + items: + items: + properties: + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + mode: + nullable: true + type: integer + path: + nullable: true + type: string + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + secret: + nullable: true + properties: + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + serviceAccountToken: + nullable: true + properties: + audience: + nullable: true + type: string + expirationSeconds: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + quobyte: + nullable: true + properties: + group: + nullable: true + type: string + readOnly: + type: boolean + registry: + nullable: true + type: string + tenant: + nullable: true + type: string + user: + nullable: true + type: string + volume: + nullable: true + type: string + type: object + rbd: + nullable: true + properties: + fsType: + nullable: true + type: string + image: + nullable: true + type: string + keyring: + nullable: true + type: string + monitors: + items: + nullable: true + type: string + nullable: true + type: array + pool: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + user: + nullable: true + type: string + type: object + scaleIO: + nullable: true + properties: + fsType: + nullable: true + type: string + gateway: + nullable: true + type: string + protectionDomain: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + sslEnabled: + type: boolean + storageMode: + nullable: true + type: string + storagePool: + nullable: true + type: string + system: + nullable: true + type: string + volumeName: + nullable: true + type: string + type: object + secret: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + optional: + nullable: true + type: boolean + secretName: + nullable: true + type: string + type: object + storageos: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + volumeName: + nullable: true + type: string + volumeNamespace: + nullable: true + type: string + type: object + vsphereVolume: + nullable: true + properties: + fsType: + nullable: true + type: string + storagePolicyID: + nullable: true + type: string + storagePolicyName: + nullable: true + type: string + volumePath: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + type: object + ttlSecondsAfterFinished: + nullable: true + type: integer + type: object + syncInterval: + type: integer + type: object + status: + properties: + commit: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + event: + nullable: true + type: string + hookId: + nullable: true + type: string + jobStatus: + nullable: true + type: string + lastExecutedCommit: + nullable: true + type: string + lastSyncedTime: + nullable: true + type: string + observedGeneration: + type: integer + secretToken: + nullable: true + type: string + updateGeneration: + type: integer + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true +{{- end -}} diff --git a/charts/fleet-crd/103.1.13+up0.9.15/values.yaml b/charts/fleet-crd/103.1.13+up0.9.15/values.yaml new file mode 100644 index 0000000000..d41d3a2444 --- /dev/null +++ b/charts/fleet-crd/103.1.13+up0.9.15/values.yaml @@ -0,0 +1 @@ +# This file is intentionally empty diff --git a/index.yaml b/index.yaml index 983e32a3c8..de76e907b4 100755 --- a/index.yaml +++ b/index.yaml @@ -2593,6 +2593,23 @@ entries: urls: - assets/fleet-crd/fleet-crd-104.0.0+up0.10.0.tgz version: 104.0.0+up0.10.0 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/release-name: fleet-crd + apiVersion: v2 + appVersion: 0.9.15 + created: "2025-02-25T20:32:35.297819704-03:00" + description: Fleet Manager CustomResourceDefinitions + digest: f1db884b6121e13483da5b8e46ede45b763ce4594769caf3aca2c768eada2e39 + icon: https://charts.rancher.io/assets/logos/fleet.svg + name: fleet-crd + urls: + - assets/fleet-crd/fleet-crd-103.1.13+up0.9.15.tgz + version: 103.1.13+up0.9.15 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/hidden: "true" diff --git a/release.yaml b/release.yaml index 16aa1e4d2f..cdd0f33621 100644 --- a/release.yaml +++ b/release.yaml @@ -2,3 +2,5 @@ fleet: - 103.1.13+up0.9.15 fleet-agent: - 103.1.13+up0.9.15 +fleet-crd: + - 103.1.13+up0.9.15 From 8c19d6d29e9bd2ca08767e2e8ab2f931024a217d Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 20:33:45 -0300 Subject: [PATCH 5/8] fp: rancher-monitoring 103.2.2+up57.0.3 --- .../rancher-monitoring-103.2.2+up57.0.3.tgz | Bin 0 -> 480270 bytes .../103.2.2+up57.0.3/.editorconfig | 5 + .../103.2.2+up57.0.3/.helmignore | 29 + .../103.2.2+up57.0.3/CHANGELOG.md | 47 + .../103.2.2+up57.0.3/CONTRIBUTING.md | 12 + .../103.2.2+up57.0.3/Chart.yaml | 148 + .../103.2.2+up57.0.3/README.md | 1080 ++++ .../103.2.2+up57.0.3/app-README.md | 46 + .../charts/grafana/.helmignore | 23 + .../charts/grafana/Chart.yaml | 39 + .../103.2.2+up57.0.3/charts/grafana/README.md | 770 +++ .../grafana/dashboards/custom-dashboard.json | 1 + .../charts/grafana/templates/NOTES.txt | 55 + .../charts/grafana/templates/_config.tpl | 171 + .../charts/grafana/templates/_helpers.tpl | 305 + .../charts/grafana/templates/_pod.tpl | 1296 ++++ .../charts/grafana/templates/clusterrole.yaml | 25 + .../grafana/templates/clusterrolebinding.yaml | 24 + .../grafana/templates/configSecret.yaml | 43 + .../configmap-dashboard-provider.yaml | 15 + .../charts/grafana/templates/configmap.yaml | 15 + .../templates/dashboards-json-configmap.yaml | 38 + .../charts/grafana/templates/deployment.yaml | 53 + .../grafana/templates/extra-manifests.yaml | 4 + .../grafana/templates/headless-service.yaml | 22 + .../charts/grafana/templates/hpa.yaml | 52 + .../templates/image-renderer-deployment.yaml | 131 + .../grafana/templates/image-renderer-hpa.yaml | 47 + .../image-renderer-network-policy.yaml | 79 + .../templates/image-renderer-service.yaml | 31 + .../image-renderer-servicemonitor.yaml | 48 + .../charts/grafana/templates/ingress.yaml | 78 + .../grafana/templates/networkpolicy.yaml | 61 + .../grafana/templates/nginx-config.yaml | 94 + .../templates/poddisruptionbudget.yaml | 22 + .../grafana/templates/podsecuritypolicy.yaml | 45 + .../charts/grafana/templates/pvc.yaml | 41 + .../charts/grafana/templates/role.yaml | 32 + .../charts/grafana/templates/rolebinding.yaml | 25 + .../charts/grafana/templates/secret-env.yaml | 14 + .../charts/grafana/templates/secret.yaml | 16 + .../charts/grafana/templates/service.yaml | 61 + .../grafana/templates/serviceaccount.yaml | 17 + .../grafana/templates/servicemonitor.yaml | 68 + .../charts/grafana/templates/statefulset.yaml | 58 + .../templates/tests/test-configmap.yaml | 20 + .../tests/test-podsecuritypolicy.yaml | 32 + .../grafana/templates/tests/test-role.yaml | 17 + .../templates/tests/test-rolebinding.yaml | 20 + .../templates/tests/test-serviceaccount.yaml | 12 + .../charts/grafana/templates/tests/test.yaml | 53 + .../charts/grafana/values.yaml | 1315 ++++ .../charts/hardenedKubelet/.helmignore | 23 + .../charts/hardenedKubelet/Chart.yaml | 15 + .../charts/hardenedKubelet/README.md | 90 + .../hardenedKubelet/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedKubelet/values.yaml | 166 + .../charts/hardenedNodeExporter/.helmignore | 23 + .../charts/hardenedNodeExporter/Chart.yaml | 15 + .../charts/hardenedNodeExporter/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedNodeExporter/values.yaml | 166 + .../charts/k3sServer/.helmignore | 23 + .../charts/k3sServer/Chart.yaml | 15 + .../charts/k3sServer/README.md | 90 + .../charts/k3sServer/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../k3sServer/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../k3sServer/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/k3sServer/values.yaml | 166 + .../charts/kube-state-metrics/.helmignore | 21 + .../charts/kube-state-metrics/Chart.yaml | 32 + .../charts/kube-state-metrics/README.md | 85 + .../kube-state-metrics/templates/NOTES.txt | 23 + .../kube-state-metrics/templates/_helpers.tpl | 196 + .../templates/ciliumnetworkpolicy.yaml | 33 + .../templates/clusterrolebinding.yaml | 20 + .../templates/crs-configmap.yaml | 16 + .../templates/deployment.yaml | 314 + .../templates/extra-manifests.yaml | 4 + .../templates/kubeconfig-secret.yaml | 12 + .../templates/networkpolicy.yaml | 43 + .../kube-state-metrics/templates/pdb.yaml | 18 + .../templates/podsecuritypolicy.yaml | 39 + .../templates/psp-clusterrole.yaml | 19 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/rbac-configmap.yaml | 22 + .../kube-state-metrics/templates/role.yaml | 215 + .../templates/rolebinding.yaml | 24 + .../kube-state-metrics/templates/service.yaml | 49 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 126 + .../templates/stsdiscovery-role.yaml | 26 + .../templates/stsdiscovery-rolebinding.yaml | 17 + .../templates/verticalpodautoscaler.yaml | 44 + .../charts/kube-state-metrics/values.yaml | 491 ++ .../kubeAdmControllerManager/.helmignore | 23 + .../kubeAdmControllerManager/Chart.yaml | 15 + .../charts/kubeAdmControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../kubeAdmControllerManager/values.yaml | 166 + .../charts/kubeAdmEtcd/.helmignore | 23 + .../charts/kubeAdmEtcd/Chart.yaml | 15 + .../charts/kubeAdmEtcd/README.md | 90 + .../charts/kubeAdmEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../kubeAdmEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmEtcd/values.yaml | 166 + .../charts/kubeAdmProxy/.helmignore | 23 + .../charts/kubeAdmProxy/Chart.yaml | 15 + .../charts/kubeAdmProxy/README.md | 90 + .../kubeAdmProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmProxy/values.yaml | 166 + .../charts/kubeAdmScheduler/.helmignore | 23 + .../charts/kubeAdmScheduler/Chart.yaml | 15 + .../charts/kubeAdmScheduler/README.md | 90 + .../kubeAdmScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmScheduler/values.yaml | 166 + .../charts/prometheus-adapter/.helmignore | 21 + .../charts/prometheus-adapter/Chart.yaml | 28 + .../charts/prometheus-adapter/README.md | 160 + .../prometheus-adapter/templates/NOTES.txt | 9 + .../prometheus-adapter/templates/_helpers.tpl | 113 + .../templates/certmanager.yaml | 76 + .../cluster-role-binding-auth-delegator.yaml | 20 + .../cluster-role-binding-resource-reader.yaml | 20 + .../cluster-role-resource-reader.yaml | 24 + .../templates/configmap.yaml | 97 + .../templates/custom-metrics-apiservice.yaml | 34 + ...stom-metrics-cluster-role-binding-hpa.yaml | 24 + .../custom-metrics-cluster-role.yaml | 17 + .../templates/deployment.yaml | 143 + .../external-metrics-apiservice.yaml | 34 + ...rnal-metrics-cluster-role-binding-hpa.yaml | 20 + .../external-metrics-cluster-role.yaml | 21 + .../prometheus-adapter/templates/pdb.yaml | 23 + .../prometheus-adapter/templates/psp.yaml | 66 + .../resource-metrics-apiservice.yaml | 34 + ...resource-metrics-cluster-role-binding.yaml | 20 + .../resource-metrics-cluster-role.yaml | 23 + .../templates/role-binding-auth-reader.yaml | 21 + .../prometheus-adapter/templates/secret.yaml | 17 + .../prometheus-adapter/templates/service.yaml | 27 + .../templates/serviceaccount.yaml | 18 + .../charts/prometheus-adapter/values.yaml | 277 + .../prometheus-node-exporter/.helmignore | 21 + .../prometheus-node-exporter/Chart.yaml | 25 + .../charts/prometheus-node-exporter/README.md | 97 + .../templates/NOTES.txt | 29 + .../templates/_helpers.tpl | 236 + .../templates/clusterrole.yaml | 19 + .../templates/clusterrolebinding.yaml | 20 + .../templates/daemonset.yaml | 309 + .../templates/endpoints.yaml | 18 + .../templates/extra-manifests.yaml | 4 + .../templates/networkpolicy.yaml | 23 + .../templates/podmonitor.yaml | 91 + .../templates/psp-clusterrole.yaml | 14 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/psp.yaml | 49 + .../templates/rbac-configmap.yaml | 16 + .../templates/service.yaml | 29 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 71 + .../templates/verticalpodautoscaler.yaml | 40 + .../prometheus-node-exporter/values.yaml | 530 ++ .../charts/rke2ControllerManager/.helmignore | 23 + .../charts/rke2ControllerManager/Chart.yaml | 15 + .../charts/rke2ControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2ControllerManager/values.yaml | 166 + .../charts/rke2Etcd/.helmignore | 23 + .../charts/rke2Etcd/Chart.yaml | 15 + .../charts/rke2Etcd/README.md | 90 + .../charts/rke2Etcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Etcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Etcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Etcd/values.yaml | 166 + .../charts/rke2IngressNginx/.helmignore | 23 + .../charts/rke2IngressNginx/Chart.yaml | 15 + .../charts/rke2IngressNginx/README.md | 90 + .../rke2IngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2IngressNginx/values.yaml | 166 + .../charts/rke2Proxy/.helmignore | 23 + .../charts/rke2Proxy/Chart.yaml | 15 + .../charts/rke2Proxy/README.md | 90 + .../charts/rke2Proxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Proxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Proxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Proxy/values.yaml | 166 + .../charts/rke2Scheduler/.helmignore | 23 + .../charts/rke2Scheduler/Chart.yaml | 15 + .../charts/rke2Scheduler/README.md | 90 + .../rke2Scheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Scheduler/values.yaml | 166 + .../charts/rkeControllerManager/.helmignore | 23 + .../charts/rkeControllerManager/Chart.yaml | 15 + .../charts/rkeControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeControllerManager/values.yaml | 166 + .../charts/rkeEtcd/.helmignore | 23 + .../charts/rkeEtcd/Chart.yaml | 15 + .../103.2.2+up57.0.3/charts/rkeEtcd/README.md | 90 + .../charts/rkeEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeEtcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeEtcd/values.yaml | 166 + .../charts/rkeIngressNginx/.helmignore | 23 + .../charts/rkeIngressNginx/Chart.yaml | 15 + .../charts/rkeIngressNginx/README.md | 90 + .../rkeIngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeIngressNginx/values.yaml | 166 + .../charts/rkeProxy/.helmignore | 23 + .../charts/rkeProxy/Chart.yaml | 15 + .../charts/rkeProxy/README.md | 90 + .../charts/rkeProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeProxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeProxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeProxy/values.yaml | 166 + .../charts/rkeScheduler/.helmignore | 23 + .../charts/rkeScheduler/Chart.yaml | 15 + .../charts/rkeScheduler/README.md | 90 + .../rkeScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeScheduler/values.yaml | 166 + .../charts/windowsExporter/.helmignore | 21 + .../charts/windowsExporter/Chart.yaml | 17 + .../charts/windowsExporter/README.md | 42 + .../scripts/configure-firewall.ps1 | 31 + .../windowsExporter/templates/_helpers.tpl | 216 + .../windowsExporter/templates/config.yaml | 14 + .../windowsExporter/templates/daemonset.yaml | 200 + .../windowsExporter/templates/podmonitor.yaml | 91 + .../templates/scriptConfig.yaml | 14 + .../windowsExporter/templates/service.yaml | 32 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 75 + .../charts/windowsExporter/values.yaml | 366 ++ .../files/ingress-nginx/nginx.json | 1445 +++++ .../request-handling-performance.json | 963 +++ .../cluster/rancher-cluster-nodes.json | 793 +++ .../rancher/cluster/rancher-cluster.json | 776 +++ .../files/rancher/fleet/bundle.json | 246 + .../files/rancher/fleet/bundledeployment.json | 219 + .../files/rancher/fleet/cluster.json | 484 ++ .../files/rancher/fleet/clustergroup.json | 468 ++ .../rancher/fleet/controller-runtime.json | 454 ++ .../files/rancher/fleet/gitrepo.json | 325 + .../rancher/home/rancher-default-home.json | 1290 ++++ .../files/rancher/k8s/rancher-etcd-nodes.json | 687 +++ .../files/rancher/k8s/rancher-etcd.json | 669 ++ .../k8s/rancher-k8s-components-nodes.json | 527 ++ .../rancher/k8s/rancher-k8s-components.json | 519 ++ .../rancher/nodes/rancher-node-detail.json | 805 +++ .../files/rancher/nodes/rancher-node.json | 792 +++ .../performance/performance-debugging.json | 1652 +++++ .../rancher/pods/rancher-pod-containers.json | 636 ++ .../files/rancher/pods/rancher-pod.json | 636 ++ .../workloads/rancher-workload-pods.json | 652 ++ .../rancher/workloads/rancher-workload.json | 652 ++ .../delete-workloads-with-old-labels.sh | 14 + .../103.2.2+up57.0.3/templates/NOTES.txt | 4 + .../103.2.2+up57.0.3/templates/_helpers.tpl | 467 ++ .../templates/alertmanager/alertmanager.yaml | 191 + .../templates/alertmanager/extrasecret.yaml | 20 + .../templates/alertmanager/ingress.yaml | 78 + .../alertmanager/ingressperreplica.yaml | 67 + .../alertmanager/podDisruptionBudget.yaml | 21 + .../templates/alertmanager/psp-role.yaml | 23 + .../alertmanager/psp-rolebinding.yaml | 20 + .../templates/alertmanager/psp.yaml | 47 + .../templates/alertmanager/secret.yaml | 35 + .../templates/alertmanager/service.yaml | 68 + .../alertmanager/serviceaccount.yaml | 21 + .../alertmanager/servicemonitor.yaml | 84 + .../alertmanager/serviceperreplica.yaml | 49 + .../templates/exporters/core-dns/service.yaml | 24 + .../exporters/core-dns/servicemonitor.yaml | 58 + .../kube-api-server/servicemonitor.yaml | 57 + .../kube-controller-manager/endpoints.yaml | 22 + .../kube-controller-manager/service.yaml | 29 + .../servicemonitor.yaml | 69 + .../templates/exporters/kube-dns/service.yaml | 28 + .../exporters/kube-dns/servicemonitor.yaml | 71 + .../exporters/kube-etcd/endpoints.yaml | 20 + .../exporters/kube-etcd/service.yaml | 27 + .../exporters/kube-etcd/servicemonitor.yaml | 75 + .../exporters/kube-proxy/endpoints.yaml | 20 + .../exporters/kube-proxy/service.yaml | 27 + .../exporters/kube-proxy/servicemonitor.yaml | 63 + .../exporters/kube-scheduler/endpoints.yaml | 22 + .../exporters/kube-scheduler/service.yaml | 29 + .../kube-scheduler/servicemonitor.yaml | 69 + .../kube-state-metrics/validate.yaml | 7 + .../exporters/kubelet/servicemonitor.yaml | 246 + .../exporters/node-exporter/validate.yaml | 3 + .../templates/extra-objects.yaml | 4 + .../grafana/configmap-dashboards.yaml | 24 + .../grafana/configmaps-datasources.yaml | 81 + .../alertmanager-overview.yaml | 616 ++ .../grafana/dashboards-1.14/apiserver.yaml | 1772 ++++++ .../dashboards-1.14/cluster-total.yaml | 1882 ++++++ .../dashboards-1.14/controller-manager.yaml | 1196 ++++ .../grafana/dashboards-1.14/etcd.yaml | 1229 ++++ .../dashboards-1.14/grafana-overview.yaml | 635 ++ .../grafana/dashboards-1.14/k8s-coredns.yaml | 1534 +++++ .../k8s-resources-cluster.yaml | 3088 ++++++++++ .../k8s-resources-multicluster.yaml | 24 + .../k8s-resources-namespace.yaml | 2797 +++++++++ .../dashboards-1.14/k8s-resources-node.yaml | 1026 ++++ .../dashboards-1.14/k8s-resources-pod.yaml | 2469 ++++++++ .../k8s-resources-windows-cluster.yaml | 24 + .../k8s-resources-windows-namespace.yaml | 24 + .../k8s-resources-windows-pod.yaml | 24 + .../k8s-resources-workload.yaml | 2024 ++++++ .../k8s-resources-workloads-namespace.yaml | 2189 +++++++ .../k8s-windows-cluster-rsrc-use.yaml | 24 + .../k8s-windows-node-rsrc-use.yaml | 24 + .../grafana/dashboards-1.14/kubelet.yaml | 2256 +++++++ .../dashboards-1.14/namespace-by-pod.yaml | 1464 +++++ .../namespace-by-workload.yaml | 1736 ++++++ .../node-cluster-rsrc-use.yaml | 1063 ++++ .../dashboards-1.14/node-rsrc-use.yaml | 1089 ++++ .../grafana/dashboards-1.14/nodes-darwin.yaml | 1073 ++++ .../grafana/dashboards-1.14/nodes.yaml | 1066 ++++ .../persistentvolumesusage.yaml | 587 ++ .../grafana/dashboards-1.14/pod-total.yaml | 1228 ++++ .../prometheus-remote-write.yaml | 1674 +++++ .../grafana/dashboards-1.14/prometheus.yaml | 1235 ++++ .../grafana/dashboards-1.14/proxy.yaml | 1276 ++++ .../grafana/dashboards-1.14/scheduler.yaml | 1118 ++++ .../dashboards-1.14/workload-total.yaml | 1438 +++++ .../templates/grafana/namespaces.yaml | 13 + .../_prometheus-operator.tpl | 7 + .../_prometheus-operator-webhook.tpl | 6 + .../deployment/deployment.yaml | 143 + .../admission-webhooks/deployment/pdb.yaml | 15 + .../deployment/service.yaml | 58 + .../deployment/serviceaccount.yaml | 15 + .../ciliumnetworkpolicy-createSecret.yaml | 36 + .../ciliumnetworkpolicy-patchWebhook.yaml | 36 + .../job-patch/clusterrole.yaml | 33 + .../job-patch/clusterrolebinding.yaml | 20 + .../job-patch/job-createSecret.yaml | 73 + .../job-patch/job-patchWebhook.yaml | 74 + .../job-patch/networkpolicy-createSecret.yaml | 33 + .../job-patch/networkpolicy-patchWebhook.yaml | 33 + .../admission-webhooks/job-patch/psp.yaml | 47 + .../admission-webhooks/job-patch/role.yaml | 21 + .../job-patch/rolebinding.yaml | 21 + .../job-patch/serviceaccount.yaml | 17 + .../mutatingWebhookConfiguration.yaml | 77 + .../validatingWebhookConfiguration.yaml | 77 + .../prometheus-operator/certmanager.yaml | 55 + .../ciliumnetworkpolicy.yaml | 40 + .../prometheus-operator/clusterrole.yaml | 109 + .../clusterrolebinding.yaml | 16 + .../prometheus-operator/deployment.yaml | 204 + .../prometheus-operator/networkpolicy.yaml | 29 + .../prometheus-operator/psp-clusterrole.yaml | 21 + .../psp-clusterrolebinding.yaml | 18 + .../templates/prometheus-operator/psp.yaml | 46 + .../prometheus-operator/service.yaml | 57 + .../prometheus-operator/serviceaccount.yaml | 14 + .../prometheus-operator/servicemonitor.yaml | 57 + .../verticalpodautoscaler.yaml | 40 + .../templates/prometheus/_rules.tpl | 44 + .../additionalAlertRelabelConfigs.yaml | 16 + .../additionalAlertmanagerConfigs.yaml | 16 + .../prometheus/additionalPrometheusRules.yaml | 43 + .../prometheus/additionalScrapeConfigs.yaml | 20 + .../prometheus/ciliumnetworkpolicy.yaml | 27 + .../templates/prometheus/clusterrole.yaml | 30 + .../prometheus/clusterrolebinding.yaml | 18 + .../templates/prometheus/csi-secret.yaml | 12 + .../templates/prometheus/extrasecret.yaml | 20 + .../templates/prometheus/ingress.yaml | 77 + .../prometheus/ingressThanosSidecar.yaml | 77 + .../prometheus/ingressperreplica.yaml | 67 + .../templates/prometheus/networkpolicy.yaml | 34 + .../templates/prometheus/nginx-config.yaml | 68 + .../prometheus/podDisruptionBudget.yaml | 25 + .../templates/prometheus/podmonitors.yaml | 38 + .../templates/prometheus/prometheus.yaml | 472 ++ .../templates/prometheus/psp-clusterrole.yaml | 22 + .../prometheus/psp-clusterrolebinding.yaml | 19 + .../templates/prometheus/psp.yaml | 58 + .../rules-1.14/alertmanager.rules.yaml | 305 + .../rules-1.14/config-reloaders.yaml | 57 + .../templates/prometheus/rules-1.14/etcd.yaml | 461 ++ .../prometheus/rules-1.14/general.rules.yaml | 125 + ...les.container_cpu_usage_seconds_total.yaml | 43 + .../k8s.rules.container_memory_cache.yaml | 42 + .../k8s.rules.container_memory_rss.yaml | 42 + .../k8s.rules.container_memory_swap.yaml | 42 + ...es.container_memory_working_set_bytes.yaml | 42 + .../k8s.rules.container_resource.yaml | 168 + .../rules-1.14/k8s.rules.pod_owner.yaml | 107 + .../prometheus/rules-1.14/k8s.rules.yaml | 237 + .../kube-apiserver-availability.rules.yaml | 273 + .../kube-apiserver-burnrate.rules.yaml | 440 ++ .../kube-apiserver-histogram.rules.yaml | 53 + .../rules-1.14/kube-apiserver-slos.yaml | 159 + .../kube-prometheus-general.rules.yaml | 49 + .../kube-prometheus-node-recording.rules.yaml | 93 + .../rules-1.14/kube-scheduler.rules.yaml | 135 + .../rules-1.14/kube-state-metrics.yaml | 152 + .../prometheus/rules-1.14/kubelet.rules.yaml | 65 + .../rules-1.14/kubernetes-apps.yaml | 568 ++ .../rules-1.14/kubernetes-resources.yaml | 282 + .../rules-1.14/kubernetes-storage.yaml | 216 + .../kubernetes-system-apiserver.yaml | 193 + .../kubernetes-system-controller-manager.yaml | 55 + .../kubernetes-system-kube-proxy.yaml | 56 + .../rules-1.14/kubernetes-system-kubelet.yaml | 379 ++ .../kubernetes-system-scheduler.yaml | 54 + .../rules-1.14/kubernetes-system.yaml | 87 + .../rules-1.14/node-exporter.rules.yaml | 188 + .../prometheus/rules-1.14/node-exporter.yaml | 801 +++ .../prometheus/rules-1.14/node-network.yaml | 55 + .../prometheus/rules-1.14/node.rules.yaml | 109 + .../rules-1.14/prometheus-operator.yaml | 253 + .../prometheus/rules-1.14/prometheus.yaml | 707 +++ .../rules-1.14/windows.node.rules.yaml | 301 + .../rules-1.14/windows.pod.rules.yaml | 158 + .../templates/prometheus/secret.yaml | 15 + .../templates/prometheus/service.yaml | 80 + .../prometheus/serviceThanosSidecar.yaml | 39 + .../serviceThanosSidecarExternal.yaml | 46 + .../templates/prometheus/serviceaccount.yaml | 21 + .../templates/prometheus/servicemonitor.yaml | 97 + .../servicemonitorThanosSidecar.yaml | 55 + .../templates/prometheus/servicemonitors.yaml | 47 + .../prometheus/serviceperreplica.yaml | 54 + .../rancher-monitoring/clusterrole.yaml | 135 + .../rancher-monitoring/config-role.yaml | 48 + .../rancher-monitoring/dashboard-role.yaml | 47 + .../addons/ingress-nginx-dashboard.yaml | 18 + .../rancher/cluster-dashboards.yaml | 17 + .../dashboards/rancher/default-dashboard.yaml | 17 + .../dashboards/rancher/fleet-dashboards.yaml | 17 + .../dashboards/rancher/k8s-dashboards.yaml | 31 + .../dashboards/rancher/nodes-dashboards.yaml | 17 + .../rancher/performance-dashboards.yaml | 18 + .../dashboards/rancher/pods-dashboards.yaml | 17 + .../rancher/workload-dashboards.yaml | 17 + .../exporters/fleet/servicemonitor.yaml | 53 + .../ingress-nginx/network-policy.yaml | 19 + .../exporters/ingress-nginx/service.yaml | 27 + .../ingress-nginx/servicemonitor.yaml | 49 + .../exporters/rancher/servicemonitor.yaml | 58 + .../rancher-monitoring/hardened.yaml | 147 + .../rancher-monitoring/upgrade/configmap.yaml | 13 + .../rancher-monitoring/upgrade/job.yaml | 46 + .../rancher-monitoring/upgrade/rbac.yaml | 131 + .../templates/thanos-ruler/extrasecret.yaml | 20 + .../templates/thanos-ruler/ingress.yaml | 77 + .../thanos-ruler/podDisruptionBudget.yaml | 21 + .../templates/thanos-ruler/ruler.yaml | 189 + .../templates/thanos-ruler/secret.yaml | 26 + .../templates/thanos-ruler/service.yaml | 53 + .../thanos-ruler/serviceaccount.yaml | 20 + .../thanos-ruler/servicemonitor.yaml | 82 + .../templates/validate-install-crd.yaml | 23 + .../templates/validate-psp-install.yaml | 7 + .../103.2.2+up57.0.3/values.yaml | 5433 +++++++++++++++++ index.yaml | 152 + release.yaml | 2 + 573 files changed, 105022 insertions(+) create mode 100644 assets/rancher-monitoring/rancher-monitoring-103.2.2+up57.0.3.tgz create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/.editorconfig create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/CHANGELOG.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/CONTRIBUTING.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/app-README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/dashboards/custom-dashboard.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_config.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_pod.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configSecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/headless-service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/hpa.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/ingress.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/nginx-config.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/pvc.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret-env.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/statefulset.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/config.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/nginx.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/request-handling-performance.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundle.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundledeployment.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/cluster.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/clustergroup.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/controller-runtime.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/gitrepo.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/home/rancher-default-home.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node-detail.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/performance/performance-debugging.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod-containers.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload-pods.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/alertmanager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/extrasecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingress.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/node-exporter/validate.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/extra-objects.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmap-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmaps-datasources.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/namespaces.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/certmanager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/_rules.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/csi-secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/extrasecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingress.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/nginx-config.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podmonitors.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/prometheus.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitors.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/config-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/hardened.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/extrasecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ingress.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ruler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/values.yaml diff --git a/assets/rancher-monitoring/rancher-monitoring-103.2.2+up57.0.3.tgz b/assets/rancher-monitoring/rancher-monitoring-103.2.2+up57.0.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..bc8ee547d89e63ecf27874f02411ab18c835a9de GIT binary patch literal 480270 zcmV)7K*zryiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYMcN;ghAbP%zeg%f?du*pwMN+a7o83A0D3+aQ+7BJcPMcJ{OaJ#$>ia~gS~$l zPY%YD$-kiS2kL+0pIRy`{$>2$eYGd|8~KA7;|eRvnVjw*goUDWoT+6!ODP`}G$Txs zDLSlhwj{}3I_|5)F60zl?;wOwf*3Bz6af;UGhT~~KmkIvD5hwsR3)dQ(SkzrjF+QI z@RFz{sb!M!vaA_ZtI?7aWs+$iGJ^Z2l1dOAK8iSrIW|&Cn`umuq1XtyF0w5(N8szN+wxVPtn7J z@wk6Ae7%yCT&$+(;dnfLL3`_}RiziZsHGwzQJfdO)oQBGPsJHwDqOPu%Jv*JOLOp5 zI!^a@a7Dj&pDy=ya8)%w$LUv^0_3D3EGI0ZL{4`SlyR0*1yvRT&oRR(VR%-MdLB(|PW%{r?hQqvk!kH4hD2RB08D8{Feoe(KMzp5c zO-pl_mrqreuN#Rj+pcq4bBq2?1b@3)*C7;c*Q2YtMVDvUlH_%EdM8~8yyOf{8@2EjHR_W+u%myqp_{Dx-1Q1&%Zj>aM`$llGzptsfy zBU;yH)Ar~#z1Z&xw&-&Uyw}x^*+LL0UoI$n+uL1OVT*2C)o<#)|IGQ4EjrxhwCQ^5 zk6Ki^MUUGaGI2q+LaBGasCPi9b@;R$I&FtdHzCvQaOrlav=)=LM5Hb8Xd@ck0*kgq zBFoROrIW4l8#?aR4*K3tCp$TjnV?m3Nd-ck<}I=dlxe)nlth}j6O1S#QHdFyYu0-g zeQWqllw-M^aV&DVtEEAfYw6NLLZU8+L>VttCLFkngRMVX$2fp*&1Ynx! z6}0P*r6fv@3clcS)D?QtiY<07$m)uV9GHIxbteh#!JVYNAv^l__kkET@v>KmDN~p- zA^;#MF)gNgQ3bjB_n&H zRa#tNp_cSQ_O)_lO{-o+`*%sDqG_dk%Wq#`p(rE&NX~RoyVrmJ@j`G?U63>fgnevsE-K_$=sRwrZ-DRB&0~%XEqJe=I4v?3vDgD!N#SUk9B1Q+h1GL z_Idv4(f?lJq9*bKDu6fYe-HNe_YWSn^}l;x?%nBsKgFLRIwEsi7YdneM{_RvXCVm&of<3*w3g@zXLGu!g*nzc!(sdd#@Qpr4TngO`8(;EM@kSf(znqDF>yHD z>3*H=AY_@HDN>@=B)zFM1s(MeJy~M5AV|R)&h;{&%zYxrt5siJ5+OAIzy*A4e$P=4 zP27pp-_udgogHjtx=v-A%F7HuX>MzZ9*-Z7VbM?j@)Vuaa;aI)P^r9Oo$UZd;i~53 zGP&GKACD@WUEl@jV=errs+P-2@V9!g`viz+x#zf$dcCb0=>bSfo@g@!1%xUD#n zB&!91=^-rjVqQFw$SA(|b`T0wr-1&M?=>mC+>#ZPFlBetOx+qF+S{-HP-Z$kyP(zg zM9}#vDy}6tBjS=~f)^yc@a!fSAI#jU0y7dAIS^93AfmAbA(S98pOZ{Y(cmS2`j%w1vQ-fZ%p_0W zQmN#i+r)_u8`}h|p+k1)9)t_2gys*YXz*v)Dg((7D>u*1-tWbG?>55hK4q}=e|9^s z|3n+x+1zM@9m2UkeY@PQvw5pc?VV84+787Z_u#DzgtcE6UH9(Lb#H$wbe(LAuKS-C zx`G~JD^z{hhpLrTV??uQ@MnJ${`?wSzhjk4H_v!)|Esq>kHu)m0z%Z--T#!|#IJqFJph||H#6U6}luGiPvij|-D%b!7 zOwPCTl1QhkLtyij`lJ9OCw*MG!#F!7Ms@V&^tqg(zYRuq0Y(|lFRA2W@OO_7mqg5<(t;@eIYGFh-wIw= zP!v=b#A&6_-v$~%qyxSg=C8w4NVDb^Ax+1G_TSFWEv)kjtL6E5vn~pY1yNAZcLj6@etQD+A~_ipRCzZPHG_+3TwRhWF^Zl^=cc{(%CqVA4^|_xLa(_-7$0x*r9p{Al zxw9QNYa&1Os@ex*NH2O|kU5>2`<^+8T2dwS+ifHL1BGDCdZAg%}~ZLq>2<)F+b?sb_sf zP=*=GsUVpuRwxq$D`M8fsiBZs>WPEeLc#+0l>yg8{bfNf2}8K5z)Qh;j71&{?hf-Y zVd3fMAVKA5PfwX0mMN93!-A`-bxayx>75L($&YF+OW3|BTO2PK_n*xSC}(k7 zJy1};waH}o2SP2MzOD3bx0fjdbU{{AH0a`tCpUxMhS5qKvvrkXJn8Fy(9onV5XCEA z@WtvcFvxg-8Y?v7bz6CK?Cm%)fb21BTE8a|MOX%)Fgn7dqe|Pq~D{*)ByJtUS zp^p4fzL5|Fr*2zH5FHGNA1P>t zd3c!a-;swq^3Yw0JM!?mCl5PdnLG4OPrs^tZfDreh62hT&WaV9k!DNi95f?AHkGeo z&O>xOcgwJgsDd@q_yW6q7C9|7&}J(rb}DcS8F6qIH5-f3&d{$+GQ*PO$R$89+NYco z3Cb)>04$~$CAx_!;uT?FRmlXdh*Kk`JFO|do2Q*m2lFmYl6P))w|2Apq&A;Bce{U1 zce`P4Zl<*~u`lh*2PBLk_)ka{!>Qj)8y|o5_{*3ywzJi&A&_5gB9OnT;c7TUZ>ojB zIgvZH`G+3FaEMN8hU^!g(qTsCT(5Ie!UfW7%?xK3=#pZD(f{BxwB-DvClGWvM8_pw z5WOR{gug^>_$S-L4nkDdFbfgp#BjWTT#~{BcPPAoL~MKM#KdQIB4^>+w+)-G)6U4u z&<-0O(DBG%vWB~8zKTOrb;+Ev+%SK(=Z5D3h@l}U=DLB;eyPhGfwjS?)H{H(GE69NPUwmj1yW*V+%B-lmJl=C56E$}(TIYHW0oVz=ESu3BXlv-f|rOk zqgm63Ke|LC;x1ymxjoHr1M-u))ol37pMSA&(Do#n15Vij8Ft$--woTXae;2O(3(NN zZf|q2Vi2#rwuknfcdfN;`<;z2er){#47HAJzgjSe&3r>+=<7o7yUB+`bY^HDWr~PP zT%;W&39_p=axCLETF9uS*$F#K0us_zc#P$c0!y8dd(VNrfh|du=pA6BKOI{bb z-d+t5R@LoY1igVWXh8orLyq00h%$(ShkzxMbYrp%~`fj*}#D4R-kVHL5 z|9H)hbFoxZtkjsN!EQxt0^*T*~H^F~ZQuXuj0#|}gKdoyBEVS9c2sl6!w>wfyh zgFpDM5!azXS+lrn1MC5arsxlo2B+<++j7mJ1eoa({Pbi)II&rVFdRKizW@NZl=bF9 z(n$A%K0-ejm59_(UcP$$bgIurl#PH&L)muWb6HD;>MF;|WGd)M$QL9Ss70ZvPMEo1 ziUwVA)m+0qxq)GeJgM{u_0b2}*2b+%kT7Vi`o$yZwGkRkYsH&9QiztokC_V|k+*=m zMyPxZc+&Dgn40{%U%G&bHI)Wt6o*B@uR>P_$M~6wIbrB(Neg0pJ6!50OF3N^>wvcq zOP{=YCKXk+BK`)Q*F|AANh6%U#WZYP)Y^Rm*qpxTOyi=68)v*MIdkXq=U;Y)NM_{{ z%jFc|SvJ|*-x;EEWltH#7Z+s?z>iHP=KjNTdSrk@%BsFNnyi+Jmk0!BJudFhaxmv& z0Jr}@l1n0}S`GABvsVT*0MG!y2;O?w;F?nqp{#H6GfWGTpOS)Oa5B?Jf3B&@e%Yz} zf|N5NFHrd$zuUuSEwm2i|o0TH1zD{`l>WW1x z1yK$aHBp?0mzWlKMhn=+{yN{(f{Ac9TIJtSsrW+Ra$SY9g3DG}*JnR2FNkV=F|MJV zZ}zZ#+uhZuT#KBiB;z92vPfLZ+`xxb)gGiNIBgV6J*8)g3oSbJmI2{z6?y+{IC9<5 z`0{nb4HF@UZk$Gg-E!<~{d7`yPv2p2U)MoCp}vo=yqN0?Z34we+&d+Ct;HGZwP%GJ zJsKo4Ug?Ios$P1tfVqdJXpnv}uprKe0v_>vPBkSt*Mvp?r>QSM((LUZ~gG)0uYT%t48l$G<`LQs3}}C%vfg{Tss|LxHX#X zARgXgjItsbozturG+Kb}wbKEpWbL?n$TO^^BhL0mhcw&oo-o_rHTMi_7 zZK)y1J;e<%OfguYENd;@Vdlkk-CpLs)~79V(9!^En`;oia~IyCs&41_eDtw6C;(h%o`R1)hAm-#|QrA6?SfQ`qe=B%7U24bj}@ z$ZtHLa^nd-j9*!~JEnKX^zN9x^D%7>*x_O!$N~p5zlj=JYL`Tv4+b=NIVB z>2sN)H`1yPLO~CNS}LSM6&6Yhwfc)~sM{Gu9)(QMrIq=$^s2d$6htIb6_j_*tZzTc zv?p<6nv+Xk!SP2&RDx`}mcPht1utbn3W$^ABYwpilLG?GJ3Kx*B4DuyW!)oE36fz& za&(1PVZv%F1q-)32)H`4k5-B_dWObC#uN`l$yCdd!(fBrX_)O9Hm^CAt3<<1=CmOC zJF--CYJP+j&!NP9_adqPo!D_hUGvYT5Fzrm5@41>Uj!9Gea8oy{o1*|u;g-R#%`V! z>Y4PqDaRAuINi0A-B}cThKs3r5=a$JBQL9tuY$J@AY>F^Hc#v+Sx~9O$~&$Nhv=_H z(ug0Of|7S|aydv0_3Qe&Sb-)qJ$l8jLF|IBWE6|Q=b=W(HI&F$R(Wq$vEDqfw(oNX z!fBn+(@P>&eg(i33%bZ9z9a~1A*|68@;Mr(lXUz?Zwu69$e63{JNd?h1v~T6UV4x| zdf(`!ZFBQBnjEH^18Xpc&m}u=lgFKTPC(I5$V{ST-lE?J~`Z)P%eG7 zWU8>O`;48ohN)oeo#5zYgJr9_C~Pz#I-bAe>O>G}Y=MnCOrr%m{pRpVU}n?TLF{;A z0_18*gbZgHX~5u%u5!u)*P5=XO;LmGpaw>PNqW=Z1 zuVglfjO3+KJNaB`j6wTtK zE(%K&tc7#vS%!fHpkbMz51<@Qi>}sJVaL+**jFUDPa*ubb@c*mvKLO`x2SnX13YH7tK0Z?F8MNacIfvn{gKbCk<9&T2j1I(CoOGBpcmG>lJ8azrn^UaY|a~tocr#+Mi!S$kxA8lq4oyvZ0&G~f z&Y0~nNfzXlO|Min2w5^*S}Zo6emV8R$nsl&2EW zvaA&xV9hew70Tfd0>(H;R`AL?`o>Gy1<}>aFXQ!*n{Ijte2_*u_$QUJCUW%fVE^%> zfc6~u!G8mTUn8j?R8nSDlLc`>A|-zNhT-O`H*H0D7%zg!wLvd?S%pn!J_0iuMn8?x z1mMA+et1`8rZ_JGn1a{(??hjD3CZxF<2WlK7v z>RiMz2S(gF2bWJGT9Ie>oZ|xe8fAhin&IM}uVMVCZf^SjAnWvSP1&3`?a5DdGi=++ z+dA*X-(gX7;l10B-1dI^fum12Uf%$HjgHxzADivs;?oQ}xdHkO!_%nhBdtIgc0OB8 z(cglO{vMTHVXCHRe=MV~jnC#m>Cw3&SOq1My`?Y82)!g?ite$x@Q4Xx2{3$3HS6Zw znQA!G!C!TioG;9!;2?RoXme!*VEtoBuEsscogZ-o=ID;TNL5)C_ijUz(0weC7NXb! zGUo51N@#&db5dF$Gb&DKjFwm~QHd)fS=mXyT5D~VZjHJ$2d;?6&kH{9A;}Bxkz*9@I zx>pn??~{FTOYn+_Vugsza7A(?xy|adBrD*z7*|FaRSCFhXDb6tkGW@)w78VYrP-dG zT#|xUlr7G!z%+z6DigINiK>cZUNi7)y>WLo?zYC|bF(A21jM+~BB|lEM(@%1+&XUp z5Rc5U_k8c(*oQ6@N9LKd#)dN-qHk7CI@$FNoHJX`JBG>=MpT*d5X@6JI<-cBZO!C(jFzvy#u?oi*TMJu9ietSVRKO;@lWWqLN+R zht%w;X2QV}*TC!V%cw%Lq6Xs|m!#?9mK5USZ1r%cH#gcF8=qs;)aOq0g9Z1}qW<&q z&xw7w1Px??vy0TWNfmjk1}OREm!NKgFoUq#d07{VRs|WL6h#dZL|6_$dV5?0B<%Ey zzkPf9>dgsC&=FCX7V_`S+bgdE^%pL%D5Z&wq1MB};F%J1u^{?}z$je8G#(S3qlH5E z3&KzuxGJUJgVgr`jl-FJ0X3|P*z}9w`qZEVxP3=Z&bUw{KfzSU5Dr5B=YRiy=!>6! zM(Im^;r{Z=^b7QZ{t4ZpUw-+a1zWFY&l0yj(K$+=zEwmpT)a7b9u56DWfvNwff6{U zf-m&$u4@u>vePrqL*rDk6{cvyyB1I08Fd`gq-PoI1~g1fgj|QupPs%(3CzByW`Jxk zqC)nyZ`bPI4-4LNsUe&*=N@f$zgzc*TXf-GX%2g^Hk9cXD4c6g%?;oK6-+AYvc#e_ z3-iq)(+#z^?58AB;zhjbFU+@|etK$e*-!97chuKW@N##zzcsD4?9+NwoY$g=`aErX zK&D@yKes!Ln_W+3MFlR-8CT@I!uj;yZW;$Dxhx9wmz|PQ-+T8BKXX>ljG%!(7=tjv zU|?}*@Stb9o}#wtXW+ZGnTUR3!nxfivq+&va{E=ZM`3^l7Fh>q5TMZj4TAlB=S2-I zpTDJ*4QRfN0qWIw!M7oL#zknJfHW(UM-XIX!x*hHFiQ9mq@O1p`)K@VvMuN)UoT;y&12U-GYk=|^SW>k4qE}yfroq-2BtUSQ6?2;CX!G?45bfr zpDg;r3zM)~GJX(G%q;CT4gI2i$EA|qWN2N~ASr`QIjtvWF*tHh0rwXl{ApubL*qlv zOH3I&bMHDk!D=a;4Q|ebUv6VBO28;wVYPHe0F*qPc=R^@vQbr_N@+-eE&{Ys~Y&25&1J~WhJ$o~v6cLFYL;^kQQ>baT`J`4A zai`DCWQvZp+@-sL<923o(J#>J=VvXwNXPPX9G>f;^_%hdQ({)*!e_hAUA%u2XBupk zDq6wA0L$QTgss*WJ5hz3kwZWKvxR$B!gR>jYaxuLW%_O`PWsn$9d#O_*{XSXpnY~t zV|{EGSD@Lu5>zR|tRn_41i|^rWG9eLH3{o=JU6*BVzozi=sw;QGmnj+bOIeji7S1^ z2+o%AByA$DfG7@!@wzzzNyMqG>vq9#Q&ABMu?cj3D z!S=(>R>6i+PpTb+pMUOdQL2GhckN$(Nq+tr{lNlwS(h`Aq|&wRj)0mwmZ$Ls4cxzh zX!eY%!A#I;{E)=m{@_62sv;~mnpF*pu-xA?rkl_i)()rv9%g-5GZYWbG>+0gY`&ZB zu9y#W#n=l2aM>!|U7R%BK15h87wNchtliOfy@ymG&w@KAK!!7ukAmcmn)e<%yGtp0 zf|*A@#TNt=wjf98`2-JVg?FFnx=lgXq+{;?YHlEn2r{>Vw0?XNA6(7uce-*eEQ+H7g8LB|Nt#1!vg3>&?|bGaa6U9S z!c1lv@>U7#qpqM~w39(zoeSRo2Oj zaeaxq>k7Ee%)R}b`VIS(@j?CY=5!1XlPFzhFRTaZilcYM+~ zIM_kiu+u4;j3151!JTC(-l~elEV!1RING^|P~p-sdAHi;iV0Fq^~x44(td+{uvYtX zBUulxx-CfjyVCd$aeq(h9^AUOq4N)Yzt1_&XO?hSpCnE6K`kn8+4JCeK!twebm;>k zf6JBJqu`6!Uw2JHyyH1|-|Th)IU&Wdq)tu=fqO0>rV_>6brZx0eVXihgT5==CN}H6 zqMvug*)veaJf|gfN?NH8aAJp2Kmlin8J?5R?I3fXj4qP$FkA|)A%nrVmKmOZUD?$Ka;I z$W<$Z5PI5bbOm7xwG2jQl9GL*A!_;SIgH@l25YY7Pc>SwC9N6mN6b$tvnA17>UXbS zpV%Lvy21mBq*1u#eo%!pB<}}vlN;@*Cg9*rhNx7Oqw@9hGcyd=RnWV+xxZ7Y+MA-k zE{%w*FF+GBq6f8?_NgWM!-HIOe($$g!Dl8)=!j>v*CmW_Mbk1Lx_ctvDD1c*;^-$m z?a^!7g@Y(FpWf9fod9?Ws+=2FO5e6~d&~69IOBqF`7V=Gg)}2}>H;?xM^|SA3nCzV z;TbK+)X)1xn11xhh76y0u1JG7yJ%LcwOK#63Hf@TJ+%o%gDL+;lBsH0`NoyDtLm2e z*Iv`#e)iTbOY*djd2F~6UmQ)?NC<1h{wSY*6ThdqM16MVTF})DW6Lnv<5l_Lw>Iol zx=wdcXE05kW`$XfMUMFFN0O;~QXedwV}(=n!kqXy?;UXv=hMV=+Todm2AnBDXSL$O zIxLSQT_jj!OR7kwYC-g{lZ>-V!mO`TK6L#DWD9<~+;#c_4!u->BdY|j@B*twcMwU3 z70+dSk@DQZjW|-1DW1>Uaa;`t+$4(c&z?|HA-EuUXlTc6U_l=FuTuAXj$GpCX0;MS z<68h(1soqEh^#m>L3}|D8SIq5ouI$?qVxOD&0Xhua)AbwPvJfzh?=LK(0$`D3|C_4 znEL=WMYraRIF-5|M`MBs7qXTFY?_d|6RfI{ZP01%!OZ}J9Iv;?^%egA=IC&o}(s7pX%;%D}Vo998Pl{~P4`3DUMUD>lV2K(Bf~{63 zS{N4wV9%OgnZJ_R>*r@SLdXfi&giCZ?~*{m9xWbdUjFP8@hIVZuE@;ZWe8o7SwyhI zr3^T&nR_I{TnHr*3*rLTZ4hs?37T5Qhp7YiEPMyT_Wsn00T%V4hHNsiM(u+{+qN~3 zXcO^ojAmwV15@B4v)M{x>A*@HiL1j!`&+VSb;z3r@7b9CSNBZhhJ9-<7*kLfKIwAI zqJr=Zy(afpg5q)BP@GtE%KNQxw77y7WB_>|6z{C$ZPzT!oM2|5!B5+hbi;3Q`L^-0 zt;2SQkjji-60tH`J6i0Tr;)NH$u1DTVhu&Q0b^fw-htT)b(=>C*CMLzW*dC<&|;MJ zpEqG`cLKLY-)N*C28&xhYny5$dpoyT?;97|qp&fTy!*;^k+|F4)cbaY#5P9OATf`= z9eITc&FJ^?nrpDsR_et~y&Pt#5VisF>$i11Xg^J3&NYBy$Jf>aAIMoWnhQ}XffZR; z1IldGcXkaoAdLl7WtIV#DI$yM=2duMqL$BWCu>)umIS?GetZ3>)~-Pta3gVpZYqG) znl*aP7w8#;PTDqaBtZpV%*!TQ-UEH8f0Uf*f`wCI6ntSCPHp|bOtr8W(w919MwqaZDX~TN&ai^~XLBYL_b#II6&v5ZG>_1ySQ51Qu-A_Xd*Z zeQE~INiNa|V)4CqS~!H%>Q$PDPCd^{OM-I8A;z8#OY<&=W+f4bzH=@-dkD6{$=-N* z&t#|r!*k;gEyx0l>crDV*W=&D$>YEO@qQ_<<#lHdl8ssg1}(RLq8aQOT#fmZ<&SSmp942IeoriTN*Qp^9({Y%gu2z z@eYjWDPfak&Nyf*eb9u^RNQ0$?$)LAR)^BsE5)Velmsu4;M~Z==;-AcQ1aAeO$fXO zTGrt=qL@bWGzMtv+17GYSWpSF4j59FnB|3)e3}uPXpY@R?;l2&>eJ}VI#eP-t^@_C z9RQjH7G?%^-*&y;1ZSG;M9`>2QkggOYO?F*Fsqf1U$JIkB0V0M&4?mY!U@N=3r&H+ z)}ikeoAlF+K0fqKJjFC{XkBl0`bdVRt<5&p*(mT9{%sg;j7rF|c%Y91A;-E+S)(*; zaIw-OH;9{iro-iwSa49#nJg4U%(*B@zKh7)j8w`#Fudza zO|Fa=0nbgh(xx)c(K^e}W?w;Y{xHNzuUI%BT@=YLtuZwcpFafpP=QU@z8e66)Un3t z7GukJS=EpZ(sne8uf&KjBinJ~n`QFSX-YhPLH*aPsv$x2xJif;Qu46jq_<%ZI|b=p z6;0f${_vV^C4iT)&z#ys^M;UC1qnjO*- zi+sb7s6=15B!%xZpuRo_=1#PU;cfQaa64S$7sgu!Cb-SJ|9H-~ZK*h9iZ&jMv4tRp z*o5S1I${o~;uSCWV)YlY3f%&thQYk31pH}oz9GeTL!!Nl=9m_>Ac+m+>q$}j56;Mj zNxW``G(~Ys7thkjoKn&SbID~}YiIWd_wg#*9(A8cIL>{8pJsCPa(^w@8PXAN~7 zzek?qa>#a6L$I{LG|Cpo#-L(8o2Ql|IW;DGuZbvI3XVPo zF34)&rx89@t!28V1Que72Jmhm2Fo_kFb87N7 z9`To4xi#|@LHE_wS*_Jt*++>Z1FM%xZ4T`xoGJ2Fp(QVJskb088VHwT8)0Sa)HXA% zePEFV8NxNRgh`7{TOo5%%aUudzW%)o+prqvLBt}NaMWX zH#HENHFu+$cMWVVFw79H)siq}N8;QIHPg{?hG^SNZ70FG4?cQIus)>5ctxO}YP`}& z3RU(@o#@B(@~iY94g7b+B8t^clDd*g5L|w((SJG`P4>P@$LTnoOq&=gTUFOcY*f?N zW2iNrMqxd=mi5!=tGE|bOw-i=f6{zjZQrH%OAw7imJ(RvMFL>M*sA(2^ zLolkgJQXymy7pw`AP2)E28NaO_N@CA5hXR8(=N)XKuQquKu^P2*4=VB4?Ts=Lqv_> z55VG~L=!p@l*9D`A>qjIlZ}6p7Dpo%*;Wtqg}pxZC!~LQKykq5;W~w<90()|0)3u;Oeq)~j=PWN$pys8hQbB(jBM;eOPmodsus@g6`3`&wBZeFXWCXXkN(BCG1NB66GR?zH0 z6CIhtcUz~qg&~7!N{O1MTr5UQRTd*L&vd1TmiEt}P5y1QZ{Y?xb`cjrthbNCm=SeF z2y>SWR*IBWz$b{Fhqzo)&>)kXQaFe3#*L(mj9ps{r^g+nx5#d z4#52f@Z?8|9v+l(7wwPbE}E3PXtLZJmy-keAe?$UrwEB{4UIytO~hI_WgEn1qHe5V zMI&tB1ONHWHkeEARBO4c>Y~_ciz{3CH?Ya=c|~)7#zjWVlM(hnzu*wXC2K)i&QG2{ z%L&dDyt@*<=!Qz&612L`=4Qk5tWwYFYy3*Rv5(Y%CPdFhR@k&_b3v{LoCz*v=mTa; zWPewh&Jdhg%jCRwuI=Ogf-+jxWf)Opjcpp)jaC&SMy`Uy1QpMNun?0OMh{3I5$j_? zl?``oc(wY{0THTHnlOJf%_36vFe32Rey-$e)80(hOCqkQB)jI_nK0Q-r%`);+pokK7|4P=+E3pq zfvuNg7G@I9gk+{|#~pq9yn6j~>IP34q>&29gywV%0#Qb7uI*N= z!*m3Ci`0%1I4Drl7DfEpiC5kMJLRfVMh%|88-7lOpb_KTaMkBrEY6J=-uIMTSra>K zx~*)r^XCO$P!3 za(_n;DEYSfMz~^5+=4tu?j5BHFWew)v-2e{$+>TPZUa|TS&f>(`1dAlqeM4Kf0Wj3 z#Hov#F`v3eo)kx?oXzih0>gH&6~YQ(TgDGjN^FR^bD;4KrDgxk59t zqzulY9Z=0INp2>oIUKVw=Ba`Bqyuc<>#h%DS@~JJH>9AGWp5{72s7Ro-I)hZmT1v~ zU{+VJ*c%5U#&Io%{ev^&6}M}!-LPIBg|Q%0G!y(v>W?DK#HuXBT|`roqNlZH3P;cD zx1^l$S}fKc9ygRJs#WF~ubi4kxaI|3+%(4C6KMf7M-t~HWhYq5D|m>y61?=@D4k@( zVG0U&XSi%~Ei4Tc8oh}dhpwepR>6c?V4J9Kk9ix@$hP8DHs(w0c{{Jwd!)YVe=@ga zAhjK)ps#bi^;cDeOmGG~s9w=V_O&sJMpVankhHL8SUm|Izq97=X7OHZlYzIjbe$Zi zj*|*f@1iJGuUF<_wS?7l7f_f^&oU49c_Z3aTx?vCJ*tlDp#ID0$_s~K6)4)kDftTn`;{yU(7XqHk0n0{_M-9Q~s|A4QYc$Q5O}<9QEca41 zcyY$6uo`P?{FvYx5d}W0EWfdur9}aJVBFu(H}LF`jOCt&HQs;*{#lDViL}tuV7I zKHZ3#4cIe%2N1R7xeqA?%@JWa`l%+yZM?HJdMjQ)t0_8pb@n;{*p=#hPP!AG>pc%P zC9q8XsR03uY%PgIOM>%4->fbN<_{f5Uy}deHHQs|faMOld*zrhU+;rNGmPhiJl-dh zy*&FW+aq|gH+eLlJjP`2fMj3J9zEWF_~rQE@px2SECMK_Ad^YD;KS#82M5XXy+>c# z;h8x!!?$VSWRGyWhIF@fZ5>G-qGyn=8}#d727%mRlwFty5v=>N7r#IiX*ajgc%%Ui zSSyDnOOkR%a>xwcC`0uW&Vt}ugveXa)r!{wXXH3PBD$*%F@AJ?SEljYqI^HP<7IW zC8hUIA^m9Ffcpd~qz}f8;I?%JP2=OWgKm{4r=T-|#VQHqZOD!@N%tP6;{+EKWlqUs zxI%;(IyS`If91f|EZ>?ju$pAcZ29zLb;NPlEw9%}{MmZG>hA_szqcM_p&y1hkb!Tp?4Xuy7QaQ^86qhwwK}3r zMu#>ch7)ZrPt51rO)lX@z&!!*_c zaB!~*zJfdUfdg#wsyg*`yyX2#&5w|b*xWFo?-2@OMm?E{V=PS-4fb^-cgwqMjb1(a zYh-I|g-^!8lP&^sFNlDI_{TjLH`$Jw?2DOY2bLx#HrY|7j{Mo)#D+P#4pO zivke!B5)GpWyKjh0SGuH$ewP&dE-Cn3Qto}~J}uMV#FuJ^C^(}U}W*ALT!>o2drx_)&1_W&lCH0wz2DDULEwE=UD}j$ zE$qh)Htn;${{PfK`V*|FpH{>=o}b&e8;evO?lCY_Tga)r%%%???!U+LBethG=TkJp z;wDzCAV#HV|D7zk)K^7zm|s%KJ$qIdZ|1e`xAy&IqoUba&ukqkg|nqk!)^Ic=bvu5 zjnw&J%II3!OCKH>`xpFeq@19a?E|1%XuU6ptVpIF zxEtO3#>`fr6BX_FwIyd@f~}7#~#1$n>p5w&`awzr8SIzNf~C7xWEg!!OJ$FER)1rT;79qpi0n-czr*wFde-JL12bCsF8T{mc!@>U$>XO&!S7U{a*O}s2Ehs(a7?1bU}vA4yz$y|FIKG;2b_GeN$$Gm zfuCnpeSSZi%X769T&aTOUi4C2;o`dBI6udiMBoKEpN!?TwY0^xYDq-=>A}5i<~BmO zxWX%m{y_vsb6OAs{^Lz}F3;~%exB7rXbFt+bIp~srsKX|5xQ8a&YI}cCC?$Xer`WW zP%oUDlIYWwpjxUh-|UaD>86E>mnS^zB^RrYw@3#5`TY`C70J)_zFf;ITy3|hmEf5! z_-MnsuLpFV6PXEGDK6wSKM@>KAEy3?hDr8%A^MVKAbuxVSi1S#@4ZO}EV9bKmPc{vINariX2k_uzKZ`F-AD zhQ;aLU>ogkcj)xt(CN3Q-y04c9fJs5lQyA=p*5M?UD_*~*AmjuLBMy!IrZ>NNyn<{ zNL`;9*pDJ4)R9Imth647K4XjWZxl-mVY%Dj-|YrFG4F1Rx7_027vr$`x!OCH;f`ha zuq=bPBQv*tD){4=55q@>Tu_fS89wnPFPiY6Zp8i2C1Vil0JcaA<3>u zifv;yns5iu-`W>ryP+E0&?(a|s|$#JteH1Q8e;}Hv)aUaIm%ZImo$Uu^BI;VKZWT# zOao_)7$^_Z^0?mk+$*7B_>_fBC+Xf^C+p=D#>{bYXw|WKeW9z?3;CTF*zAcTq(M5=vFaLOj>301bI#hVqSrdQJ285iV;$rMr)6KZp~QK_kpUXLRN z5hS)+pC{oBJg_12k0%HF9Ujwv?Bg*n9?1kAQ30BqGkM2R-f@(79OWHH`73agU0QU< zg50qncPz+dL(aVBZzQbuI+bl>cK-4DqW3t5ZC#PRFRADk6A3U^QLLQ{~E-zL& zligMaYg!$=H|ui8sN69scZ|w!icz^^CGJ>>J67V(ha%h-%WrJJ7nkYB-cC1_195?1IfwjqVdE%b04FBP6W`?lL{j-G#^ea*>hJsKAszxU+ff?m3OoF=({A*Aw;?g?J{6O?0xcY!Gq zG&c@Q;!^J$%}GergC5pAg7C2znqTMo4d%O^tyyywf9T72o9p;}*qdK{h5MMCcUMrY zQBkc^Qr)DcTCb>DtE$pcL%%+xc^mKu&;YvfuYVYokw+E#&M?#g4oLswsjb@=*=;t1 zvc(ys{q~O`vifa5%b~yISX@!ImL(Yu(K9YGV#a%f<#NWc$k7w?;%e?c_!rs{xg-qH zISPl18nj1CtnrXApNrpVz~((2$#BuS%w2$ry1T?LWys>KtF(#Ht9TxXoWSnF4R9}l zanDdmy;qJtH_wH;H)MUXHK{Pm&1#h;X9&{P>FcaXzScZ6?^YmWs~fR78pS#W6y5|r z^bbfUC`mG27!f;d62zzQ%5h@Uj|q936?IO&{$Hc}Ik~*91kbMJN?u=LF)HY6l=JL@ zh*A1S|HZz#FiQXU;Qk-3M-Mi4HkXN7RV3)>KCTs?%j+3s^HQCcRA$$+YJP3fQST5&~v41YFmB1O9OQ0cZ-hYpD z9`7a#^e9R0q=Y*u;Z92UwWWmb_!TNRTcDaLTIlZ04D8E`=4~#ty#Vo<%MbsYjQtHy zQ*6Oy0%r<$tLE$FIh)J#`FA}u+jx<~^k{}9F^`uCqA+DX80qzrcxvW7izee; zG&b~lfyE3jh~o!45CM)LmsFGJ@2F{%Y7*!XGQ6^TE;v-ea(XhH$BpGK~Jg|wERUuzc_Pv2NMfM-BpPx-!3a&Sj z2#Kzi9Gq^=YM6w=FHcb==%vOK8_Z^nc8r)b&^XW0oC=IazhALqjRZxZ z*%;TXrwApQuQ)X#UwSNUkv^8SU=j^9iVk))*MSD6{Qz;bB;txnf(E*8GS-5I83i-RA;P=T`{fqvS1>N)M#c@si*i$S^I5?s&y(bVcPdehrv0ZkT|`_=m7q@GMXDUzOlL zrj(ELDooLdAZII<-BdBjNkJ4zDvPq;aei@DXBiM}0o`xH$=DS6XH6|?lM3(L&>G-Snsm#Zlsv;s**ZW=h(f+9L>H03+`p7K6Hf(?yLb$w+sF?5;y~=wBV||r!ksF8OD=sdyY?|oBb#mg` zGZCBto+ti_4!&DmU1A3^>YPYZerSrD&BwLkAjq9X6<+fT!osLh)b$K7f0zq2ZRXU~ zW(P~c+e;F@XWJccFw!2fi=0*?{$zLg_#_;dIYZ#)<+(u45W(3J1f#GTup)3N)49(E zp;i?f?|UzKPBfrTPXM4;o7zGd!P z<$-SZjcqsR>$nIi*sbm>Iy28oVYVRM!5tCC6_JL?pwge`oJjDT(^Q|=iXigVkO8Cw zp3iBfOTbbAStTGtph3FiO7j=q=ecj-4%TY}%CV?V41j2SZajKb!tV|RcU`OFlU)?9 zSYN*y$h2t??-G?tUwg+VdP4<~QcEk!@~JKTp5-I$mg(K{7i|dhru)8@L?nxv=ENi7 zXy_Ivq1DF4J3fhaa|n~Lx@sf)*>orVz_<&f%eJ&sq%vY#hUl=NJhzAKAh*d^1AeFQ252a@R5k@D1azWMVNr9yVM_L8%&q{Nao29MV6{k_cCv;<;Kfukm0#_p zWB*YuaYS9Kx+omcI-bAe>O>H!QU15ItHJrBmxK{1%^Q>LJuGbLi7vc}X|_s-*Goa< zk{9_D?FWn`RTLLTq`)h$I@oVkxgvt{yscD#^{*j+QvkIQW;(ivNNtXDx_)ND6@Q$m z9emW88HJj$tV#5*&Op^8$y(K0z#tB7v*ulWgtDrhqP_9B48EE7&EfM4>c6|5I;|)D zttZer5Q6PPb(*gO-iO&&%5v1)k&U2oU)I?YVTlg^dWN3< zpc#L#>-(CK4C^lJLo!KKU`BjzvlYP1;AR5HjhdNgQ0PGu^Lv^fcyfGnYUHK1(o=#e zNw>DlV|(?NT&kCbwd|@PER8T6hGuA_;WP9lf<===B1h{jYn!L5XrbT8dc9|m6~!6I9>hms&BC#k&5QYRqgyu_5HHa}pia=A)#CQ}0| zmGyj1-v(89rMl~ySS19@c_?r-t20G|y+Lmik~2B>8VBPZ^@KA;-Ud37x%^h}x|*Ud z4-WR58H<`7ZYX)l*(v9$b&W#N8%adFjU-vd%j!h%IW2EW$FOadDV_Oc(!wD4#Bvv_18g6Zq^{E^Q(l4!5!XqUHF z8@RO@<~}4+UCl)iGtEsBl49K1BnkM7HJDvC3Je zfv(7mSH_V6OlZcP%$bh+Y-G;wqH$EX4;X<8D%FE@ry*ZdqOw*sxUYb(y`+~`KFbMP zHRDLpnW2E*0RS^Z%1W&epZ!QO>po!PHJtxT#3}*Z@pkvn;%@^Jd~cEeRAUu!q5X*s z*ac+mu9%jh-(}VA>luejw1ZlU;4YWp5Iv)XA|jZM?y}HW-HJK!`b{hJ`K#V_fdiL6 zaMEEJX>}P_GV(>IO4H|BC~Aor2T!VXJvF|1R>n925yM*Ky<;`_URnObR!l2Ggq14X zDu<)CSoc?|mYN9!Eb`tqu3ZERdRyX4mx(Ndme|+`$`(oVeVj%=&FYhU`gr{Kp;wa{ zfDf7xed^()9&dW&IT))?;~sCs6b;;)-=s%LF*ET zCa?hT6PoOep1_Z8!#RvKR1LUW$uw_Lgb-lzr+${kJHtj47^^k&2}f+JFxN=qMY^r~ zLbFsK*JUA0ts1#1bX*TuC=P5VYi;eB&{)*Fsb2f!j+O^P$NS($zk^$O99WDG{0zq4 z&mi_O7>E9JT`uzoN_7(VKegck|-RK|C^$gh3 zC&1;}>hlEn&|Ux^%mbjcEP>^J3#)(ZO>bZoZ92Gx#k}j4-(%6gaWux>4$V8(`|ZXN zL+G*8-_lCI!9u^rj=QxDca8n_7Pi}7YrGB0JqyXQQQ~PzzuR(j(EN`T!L@*)*%9#K z#2#_OurUYRA>IcJA6k;4zuD>2!=o2Z(=zX6Xrdd>${C>`VaNH7(KB|st>Bs1t54sX zb^+a*S;i5TE?n+SPtk;1l(pc!2;i-=B)Q~-K@NgIjoO?%4dW~V+qz)dJ~MYI9Pd&% z{_3H{zt0qoXHf>B_f9&rCQ{gk@w$LZtHjwDDcVUd2!DO&7 z@~m^>>38Ajy6^P3?sVOD+Acd=yX)NGs?&1Q>37lTanI><%;_~2_BiD9JL9ae{PsHG z^qYLwnS0;S@#dqL$Kjl8cDC8Zw0fgiwa=s)np3?g)vNI@iBL2Hr6VMk=DOn`NO6tT zX$Se1CpA?sJGFD1fqBJ{;yaVDuAg6_iFaJ76dT(w383<4}z1D!1Z)L>0= z8DlWK@127jt15ynO|+-873*4YPY>M55N@Xsd5a;IpEDAIO1*i}f0B(2(VPxLnUkPO zQV{`30hoVP!fW%I^aCIhz$-axHYUT&F#st#3cN(Jsy6x)V0sKXn0f`tPa8)d&rhCN zvy`;=w;O~dbHVa9aHhcQXr2oM6E6JkLzaVZ;jBJEqwj6nflgJo43>?sA^qkkkUUrw zhXgJdro?X#O{IXI#{A;2#r<`AqHLR@UgCPtVn1&ip{8+k^`!3fZ+QJA?0wx!}=C6OJ zP3(gH=P5CIslhJvKhT6{;9RFn&o6jRrbrib^RRfGql*Osx*WLg-8#yrpEUMF>4c7Z zo1$}~0h^(@mzrWytZ}H}{QFdV!=Sb$6yIP;j$yh%S)lSGuuauvGL;Aqu&>X zv^JDzO(4=i7>9Jb7^Z_pNAVi`X^^BjNiyt^4(PDwI=zj_INuy^NyJQT`&UG&_8+E` zoz|X#6;mN>H227fU>fb`#!=}?i(Vw}q%ZP64;O(!uIo$G2=ce>F*>Ome4gIDhrq3= z-^pn$=w}ehxdF7vl_MK#jQ$C&0+?E|`R`wM1dt0NviV}Sue9Z`ktlvan60@aMVZRw z$U4QPmlHSU59C~Lc3@jakHFvIk{5N^xEes_;P1`1z{{Z10GafLJj+o<#|A}_fDeH`T9nG>vPK_tqF%mkh3%YYR83f(8^ zA~mX-UG!5;#A??dE@Z6~q|{I}_1m>L_n=9k1m01aarHg-sntGU62+WIx?mxXkU-zY z7;3g$|2Aw_s2dC|<*^UTX7jxB0P1*ydDc7x)Z?S}&*f=m-BZpwkLEcaH22#^o;atp zR(aA$H@;%0q~dwEpZ+(p5Im68;Vd`1vl-pxXYI$<<6O2S$@6nJcx-NX=cGhs#xIFj zAw>GJZUsSSGP~t8<`hHHO{qUcWvGL8AK_!EN?=qg2yJ_$10s zAAL3*?FAiu=Sv2tG2;iK+E@8AJI4`su;ZEEA!As7_z>F>Q z1%=OzC4ap~=x*TEh@4lW^X?1xA@l)tYpG}SGWxz35s2XF25V|RvnQ%a=|4m-3CU%Q z;8tDa5|d(yu1Ha|kHMzrLC`vnnA< z?FXo8?HhXOYpHoqDi!6ZeEs~)IUks=f^mBgQ}eKO1>tVq%E(n+JlDb+IGB~EeT-^H32o^-ZP4f&)B5pkI?QPNTX5qlUJKVNwL+57n z#1li}5rEjf5Sv`Xa9PX1RWQsS4ELG8jcNriONF5&?W6J3_WiCm1$s{(=HEJX{_Wk6 zk@NKyg#TS>`wf8&A3C0)nMSLl;HD_3RD>bU!Z{htdOUs{cqHlN{>^&BZ;4*F%5K>gZ4zu>UvkIWCwBZDMP|Gc-@I*f zPMGR*L);8PaRjO<@dCFIIXSM?(mpTkAS@$6VS+2_b)8;u z@4sA5Mnk*7XBDwmW<4vYT)yP$R1fjcQH!>g=-8$d$4q%SLe|1M>t>uW%u)D!kWyt<^Z?3Y%4I41Kdw_@kW5&@xp0V<5pMCr2p!%=D6b%B;bm;WzRR< zNx#?2C(?&kJV&|DsgGumRKHibzNO}Lb0Yttahh8}@Oe`4yy@F?9prn61#2o1Ez4Sg z?vk_2MvFVI5emSL)JwlYXsCB_0Pa)no^_4Lhn*-Q?rr3^J{b|H;{ zF}h-O%_yE^)ELKZg=!koYrQ!4bE1eSAxdq@ulx+5tH6P*ajLkcEAlt+OQGm{9% z5kYF>_?!_Vx&w+<$Pd6W2O>s>L=_D`Z*a%|+=Mt#$KEYZH7j9 z*AiX<0=qLa?1vqffpQ<@NEIn*J%lBeOOm4*mNY|J775)39yq(b8G{c)r9l)L1ortT zx&NQNcU^De#u7#Qx1Iur_H0WT@urI%Yft(+YTHRPZ6|6=?(|G2qZC;nSy8c2sRAXd zZSC*8#(BN-B7S?rP1bhw0IeHwV+3@j?-+s6MkDF2YczpSK@a^|6k1xqT{_7{2D?f770)G0%e(*eu3e74O=1z#60B!lh3QkJwJiP@A_{b4%Zl(&b3$w-Fxm$7x3?BOeS`!& zWf11UO0P-?t@dlzZuVCRDo9iAw;)7yI0+RY=FON{7X?C=$FsbSX0e7VvcNU2}-8i4~M&v zl`+gb*rELXg$~T(B=%*y?DvwX`jtiWmm#1l(D*Vk}m~NVKPm6Di=RCDG`;~19 zy`IQDwv@gsV9&iq-C5qoXcMnz6R+nF60c`tPPeK@!>6~c+e9+@tRfk?htn&c>QA3f z>a*2huHP*&#t7)_2=(P_LUPMmnk`aC5`D9I?qP?dT5lmyugt7o-TIhvTUE?oFyhhtao%R2BH%v4^3^}{^4 zl%c%}_LRz;1;vD4~_q<|aOn8GwC{a+*P2ea^dlet%3CcFfGv> zlmwBn3?*q2BqucDvHF*!LD%Hv(O6t@Kygm{;QkpUQ?|MKj~JnZ#IZ@25ejR*tnRym zbH;8HGWkE5SnK%)#5}BZX;fx@Xp;(Ax6aGf*A3oKRLr_|YdGP)$`WJ>!I)j0{_K3~@ zt@Nli34nGJF=Y{QlS(IS9kQwI4waxTB&fczA0t0%`>@;{nT!)~&+r!5s(COn5D z(vk5T_Iihdo{aiqN8xCZJM}Iyr%K6&Cu`~>R7{!*nks+JRQTjcnF{aAGp6$8t&+0r z%6Lyt*eXg%Or|#|PZ2oNgmKBpG4fa?IrnvP&sC*Al&1M z-ruGE-(C-J!RMNRau`z7-TkRyG9{)%cnuPbeSiwKxAF#=#1`^+aDOG_7agt~1Eda}-P{&QQd|3|fiJf+DuA|5Ao0YddG4{vM~)Xts7%`FT7K(!6hV zbF5%kJ%yMlXDJ?AH_qRABo#8Bv$2iYW@AT!IFTUYCCq4nZNF>I1Q5%m51Z&3ytFnW zyRAj>uE;2d$aSLUsOKd81lCHdSX%-977Pk?{1<)}EMlF@v5++xXdYMRhzW-hc>!va zm?h#%llcCtj$(-vAB%9&KLh&FaET|FJ3n`GrBP3vh$Jme$x8n9{tIB(> z9pxfcI0z09!t&FZk~^oS^_%c~5RC!csOhXAKGYv>)$(K{ z!GTtWRoz1K!Ll-;YTD>oZtz|PoaI5=Izl_y1p}-OTNU=J?w+|OOO|3uD0*?Xb&W8@ zb*VJ&d}-8q6R+y8UE$`t{8(Cpn2{Lk-D(e6{pb`oDuZ|A^k?O5YMZ}2*bQsv>+d#SCW#uzeL#oE90lQ`()p~V#8eDlnEziy>WZW^-XfDIgYOpKr z#VRyD)jrI~=;3m-fSrH$nRt6Po@L5<%E7G6wvHUTd=rO8_oC5KtHx^P6K*etM`35N zm&;ueM|kD^7xMFEo4EvWYQ&Bh_k-QZImEs9BHiF(dX3{!QlbVEo zoW6Q{^5)eK=f|&)Omjls1dqN=vuL!SSb3C?gp#|&mZvxXGpBqUKp@tTQ7-{W6IeQ+#idzEB)KV$>j&!cHK$i}mX;xS!hxF4iZcTAEie)i;G zY%SVXhnM&*Q|Yr+XeQbKB9LYz4W%3cL z=yOU!6eKk2s{FL$AVFz{Sq}v#$~D&+-;-$^jBQ1mLAFf=q{w@sG=`QrB4b7-w1eh2 zNQ{Jvx-H?_{8b9ECBNd4-l!^o=rvKc5p1|P0kc{MLX!|QT?AQ>Pz-URWKa+bHGvY_ zlocpNIr>Vb$6X2`Uz;6H2ZaSAPb6tghcwepi1UHR7LnEpdRr?=%5g#?9gNa8%&?3zAoyJ!+b@gl=A748Vrpw#N^k*^cyrs14j zQ-(p%Oklg@%PQ57sJ_1R%i45olY+S>1+%$5_#u4(7M_d|uZ-YwU#fnN6vIjB`}wQ4 zI0!9Ez(p2wM2`A>US!y3^(3O&@#d`p0CpWls0$66uOCkp%WlFLHh!c&uyF&VoPh%) zz1opW7x}Dg~ypCb*leT(Bbd0U=kz&3WIz$BEZwa$-(vwJN;}4lg&~J zBXqZyFBK^F0wfdvie!Z7f2ov)_WeU9KNMF)J9VK>Qn)`D=nRJO1TCSZc`DlsujPO2F6-`tX;*Jn+NC@r-9kzvGZTy&w zapxDaq%+%!T-uYTsi|fFW(br_AX(6KK^eOae$8iC#9oN>-AV?{*pdf9qQfF}XJn4mQdIcpyq~hPv1)ic zCAEB|b(vo0sH6@L;NCU-iTEQ}R-QNSVE=1RugzqJt-EgKKonD>L7!O?(Lgw!s%TTD z@2TnV$HOF8R6p;@AR+c^MN)bCXDh~%T~dd0l*w|9CZ%wwhyIg{+=?JNfbdQbr;B8B z)oN}?*o2&%OeWDdvU?Po8(;Y`uTT8ON9=;X?V)3548gup;VedJS7+b^KLMR6--<^c zL8>c=G|@ApGF%`BE(X$jY#!X`oH*c+!ugJYYHo1YLAC1WyA0~#6WHj0CRrX=EWU5=2^0JqYGvEf+u z>ejWmG{^{$fi`W;_zx;*Z-{aV4$T&+;#`k!(W@91JfSDtlO|CTB7WKDx$bs#kVrbi zH+rE$3X>^a=7%l@sWpIewy?@^^pz)*rBG#cclqAt)3f#CC`im7u!>C{Z)1$&>=hIWd4&yjt$~&s1gL9z^Ew~C=2Nk z1HcZ89Ak)t2udOFx%qWE#s+!gk&}cH1tZokNyJLY=||adlbm+I=Z2li8I}1gU~KI6 z3~Kn{L&j62vPRL-H>k(e@V)6cg#pfQ^;+VQU4P>au;Q;_$7JhSjU&|g@B!f6-dZ0; zGLqq7V78Bw&}^b~p`u=|w`B3^6uStsuyw95 zuscDK=rC+#V*BO(Mn=kj2I5C9W>cX7H?Riya@6Bwxhd^Pwxk@f?Xly%1mZ z*m^tq`p_ifes6G|B4LaZ${>XJ%HnaR7e?5%C%aEfXd(-uEAAIYHD;g+b7hyD4%QZ3 z?u^)FmpeOy8w00qx-&di*}~!*Z3^I8(S3qr#xwi%>%(*8c+M92gGa_K23K;3tB6Th zE*~=2Hml$S3e>WpGSl!EjNs@izem69^ zgcxgS#huXPX2jSU1&Pre5M_HuI2n4%LsIHVzt9;6QKJBq`fgM~E6bMnAB0%ivr<2A z&4{H$KnC!BK@y7JQ#2!S2=+5wZcL@Zi3e1e;cL3iAX$#;xHmG4UirKaWe_c`JR%{ zaoFI+4P}}c8H-!=*_Q`#{>hhcK@xef0(Mx?ybvow#72ZICt%{_D#}P=*^{0ddj%|I z-lsT=U`}Fg9U0un_z|pIkfk7@@eQmIvY>1iW|$ilGKvX_trIYw6?C)^z|d?17FTT> zqhNe_9b~Ywf$(x-U((JU>-%|PGdCsze{1b*PFha6m^_2&HnZOd`FVDPrsLFnuObiH zxpGw8jE~l9E|=qiMX{QMye02HK7IS+f4+KgesS_r35eOr&;OQpW62M69JmO?9Rem9 z{RYANdN;v5HlqFg2!@J6Vg5o%%ec9g3RxoK2}X%nj5^>RDFQJ2{Rl`pejnu!0Ye@e z(Znzc+w}qI3Cgalbp;(aU`wbTs%+3^Bes6xOJDD+g+=NyrH z6W7hL(mX9zu%I)XP?1tsq-+Kw@KehOSJ6uq-rAepd69;JvhRmProbKWvg2YBXpQR; zQl!db&!E%FvnR-=hRIGB*(2CR-Kp_|hn;8Fq5o|b85=+P0D%A7W2;SlB9Q{rU$@#dX1pl+oRUiX)r}n9*%@C+!`D; z#7@qA{#bSC{ogtNW|M#!JQo2b{iEtBO}QmeUoXrToTbW6-AFwhpmO~H69thWZ5X9j9`6x zb_`VVe3p^LbawV8H_!0v83xxDn6Ury%xr9eQ@LIDBO~q263$I z!c^WWv5mANt^{XhATZWL+3-5fj<5JchpHef9XbEGVjYaGbD0jAAd*GXDADQGxX=eK zZ9EALkFuYOo#OcFozEmF?hqn;!LNfG@YeDUx41D#xFgJPa{<3B#=wV9*xOY*eYLQ$RB>2k+~Qs)K&2ZO)F?4=ol82_$uX%f<|&82h$l!EqVEgWS2Q#h0&x0UvqEt z&>JNv9T7USBhm3tjN%Os#c3tS3!+S~A%0J1*Clpn{ECY`XLsFSVS!4!{IqtoY(sND zOk+gF^n&9PPK1Yf`4-z1`%COJGqRl6^Mcp;E~8xvSz{vvK{~YvDx7C8IH!PO&A8hf_*=k1C(p>nx&dBU*${==Y`3 zd?;=gj47a$v4N5Y>Y?KpJSu5)h2tBpE>LXnat1=ag1br*o|t>%G3sDe5Qn%g=w_m!KBC7AvL$c=bIA6URcv) zBzds1AX+__$J}2rPG4puO>xL`Bj$E;#6nU;$pnPF5<63L)-XHg-X%_{&R!uqu00+N z?lHN6P1n0Dpab0=-4rREK?QBLW|2S+ixPMOizr7^I3_bA*#(4?RrxOf04^P;9vTP- z(S&dZoSt>67HgP^0p zc7UxGvs09Sy(gd(#7gOB^ojSSFXGEDKq$!9a~`p4gxCR^x<(VL1t7K8d?KmIJ29py zVD}$!M%(jqR<<|v!ONN0 zt(}Tpb|~YN8d5b85`tHXa@di(jC}BOcN_?#qJ$(}*sV5rNLtG-nZWlMdP@#R?osaG z%%|6059em?RSWLj{m`lYmx&HAAXK|3GQ8N8YOvabF zN=bEvGa8LSaUkQ(;%@ieM2V9^Or|holP|LMu)#C3-9vtL9#@0sw+OZVSj5(js3}w&XF)!b?#^(M z$mV&(i98B1J0cYS-?QW}1GLSWG(uDqDu68);R)`9lN3X1hY!U(O5U=cXS{2gl|pYt zj0BY$i*cM_7!#UC)no30o1<-+Kk)dcGxX}OXFDPfKUW~W_~AsC7zc4QCOhK3hDMl2 z1%%)*B<`khkYLr;=msE;xgCtoiz9M%r6kOspp#Sd;^gI9?j{xr**ChzqZuKWxlC4p zu1R*OvP$w>$_%3!$?20EC781B#vMJH6r>-N@bf#TtSMo;!|h+9IhZza)1zo&SrmyR zJd&3cl1bL^5295aAVdgbVNMw@874VTp3eZi;KTiGl;!lNh|a#}AW*QNgxxO3K^%)K zfdSpQq_9fIaTMgjn1^*^&bNZfo_TT#gDKr=oMV{lEu@tqxDh6dY-Kl6o|uyG|NDRb z-y8tsaM#yZLEG3#cou=eq&FP&*guA?u0p}GuI-{=*bCCM$FAbB{bgi+QDVRhqUsc_ z11SICkf8e!tLQ2M3o{wFiExP%fSZVm=lA3}C_%WaEkx6dAw@yBG9p*Fhqluo2m3>= zsz5Z2`ORMIqVBA53600B+Liky!Hjf;S3xpnh;h}Zh?d6g zv0TUL_O1CSoPl$!Cnb^GKs=x#B=u6Ujegb5|EkIUlbmbZ7AX(hD&fKQg6z0LFDC@g znykx{AAE965YqO8E;1EFF<%J%m$LEv>G(A_xrB&B>|l;DBaHCxiy&qs>*-*iPR{M&p@cV&k_E-NJcsz`o$)Tc6l=;pm5dN0 zix}tMl1xy;x8Pt4!~{r5kVc?^WfTT=I!JtQLOv*R0Y1TfX$kd9t=VZ#Y zraXqOjO!*KO)@e!bW@CV%BZQgHNCJrndL;Qg7G!j1h%2(qnL}le7cZ!3z;B zzq8xm4R2RlBi^tH2wheIKPB?cTx>#$|D}Es#FyOnM2pgaeQ=e8X&Ohi=ldgnpnox-9ZUFDPZ2#K&Vnd6-Yoq2$q( z;AXXflj=lQ_@T?p4C)JDAsc!cu+b*+jtw4@Mg(LBRLV_= zWC;fr_a!s!U}WEYuq@9r4e9*#nZr*Bc+OxN)9iz;GnY!(0{*G!Faz-77#MP^fcXQB z19v(cC_Iz{1eE|oX^QJ0;*6HSJiCli3(^(N_@6Q!v=?Fo+5I!js{ftUaEXT>{<&#PY&c(@bo- z|3}8LeauDhrySFr-oI;iOwQuH*3m6ky^D)qj9_rJV=c|%%zgYxM>Q)M&q8P2P#!4> zec?Q++_nB^yZ6=3WA(NW-NOo1S?9ndqlH-fvXN-V&nw#|YHQ<@PYXGE?wUtFCE@+@ z$0-Tb0nwUC_OO?dTQyiqV{)3am8UjWt61b*jU;SY`8e;?V%7 zmHMV;T41NIrnYlX!$hYlRq>a~K+DzQ;hyhOibg`gnpv!oxlH&eo7+j{w^Y(!Gk7a^ zGkLp+akk~seA@uOKtR7CWlZKNNpM2*ezu6gBW3H+qb&mfaADl;EhaY>Xn~&-8HE=& zBb+dl&WrVeVt^3iN*;{IWRXy}vfwFuTgB7N6M>lq`K1oOj3MT-5@8wd#TBP38tqFI z#rtFanlENf$_>Xz%e%?nsTayii4nIj%Ckkvt-lu`Xw-c})UbEiBIyDV8{7WpxSNKf z{v++|-wjy*b>qD4Eu?%3ZRS17Q8b?~z%C7uWUSK}1xYAnb)aD32h+>Hh;1X&TZ zJks}8*#>^E4w@z_8?9O1gB0yRy|Fjk;0N)d0E-S8Ext z;^1Q2C6jKD$f{fNQetB#+)B2}RyUFv2F#P|dr+B3>qZ^naDq-actcra9Y>&RF{Kjh zG7tNLk*BOD!orV<5pOu4v)sJ8iXI3{<1Qiz=9gtLSvr4~3E9XbVIssT+We}{GSO~b zFi7V&DMs5NrZ}6!aXus0YDl0dLofh%;&4Z?19!SWMwn8Zbr}*}Xqhb)#!&KC^d?mi z(qvDL&**ljp?KG;)JLYzPhes4u3Pm_ad2-GZiry|6u2q!`kZZWSzC_dr_9nd29 z2m#7Hww@Rfc=7`YF$40%Y%UQmkI??$`9LXLUJL&3MHS(hjY*m|%Q9MG2+axau_JfIBx8tUi^Gt?7m};%1m9b2?GTC*!7hx?Vt`a7gHL6u& z_i*VfxpMlAYMWWpHfWWaW}JidTX5DI|CLy*$<$hgv1B!N%Nt6r1I2<}VoPM3+a#_7 z+oA=aMi)-LcUgRd-o1UzBfkhQe@Kg`7%?S)lfqQnHCRD0_McRd)M!_F=!FX6DGuac zmHy*bGc%HoE)eB3B4~d8`b?$@#Yh?ki{%+{<#=<`yzBInz0JzTiBNCR1CR2vYI2^I!@}nsYzzYK7>j z4A8^JTl^;~KzN?3L2v({H}q){{7L9duV{Zt@ul-?JNJPi@VhmP)j79KWz-_qaTz&9 zGX2oce0ryv9QMZA?>YOR$ACZB?DdWxy0UEaC9*d=QkTHqT~({4uy>%9Ayt=?-3Lo| zSq)Z6a&~wxJ=eGsMn^DcT@Z72*s`!+b@yZ{01Hn=<*O1siZqX&G_j~Et7Bub#8lm$m&O@n9iOj!m<0K#Ve3$;x(Bfo+DY*dzWL8EE#Q!i~SCp zuHSNYJtEVo+=dB2F4>#I2 z6dSOuAq%ib`5jz}aIhMXV);a3BLPcWR~Ino30AF`O8q>VW0h#%mh8*|Zaf}D36Bmq zA(`?t0!icE!R*N{kGZnQsX>S`JYB>AyYk4*Alv)rp!@unf8UNspFBgOkq@^r<)WK$u}Q+k#`zXY(k+%{ zTl6Pe^qQ6&8%`m@sYB$gDXLG#BuOL#PAqp4eJ{7;Z2&PU0U{T6uu-|vM?oHq zFW5cO{emEv6`z%y%<%>LStyeRd45f@P?TXmtui?y6=OS31~M2GYXhi(vO&0+GUNvu z%`sVkO3OILU!A^v_2T&a)ytkd(%-#(EsTsHhaVQ29dp-mp;_rB^DcH*#ApShV3#KX zb|w(HA1nu)uxxBU`M{bcbyd52k;Nrj+gT{@viQi#AgvTc@7}&Pjt}IVA6mexXc4%c zYVYU-Guj(uATyFFA<&p)8RPTVxckaR6Q{vkXi~B(8L|q3u!BhVB77Pb0o z3xcKfC!8jiixD2vm}749@--o^pmL&wU@!b<5QykOCvyv-vR=}EqNtqTgMQv_3w?Rlgm6gR+b|{LvpWO zjH0+cJ#(bZ6U&YPpJ2VW1_i1e?9 zeeu!_3?UA|A4n)-M4T!Ikns$Mix^Y^emPOKK^@qWabXCX7;_rfC$A<%;G{*pijjr* zjL|$2>q{iZIyE$7ZE$0Bc0qAhHPp97dQuFeg;GTs5D#s6gboISH<9H`6(lt31{09Z z-9UKOGy@qb=hVfx3eWpA=Am7zylPx9suH&{a>B;UiWhCg{6w9cOlU{M+ESrCbX(#k z`c(??alhgQe^eDf^qRv2M(V^PWCK~E+QL~R1k2qxcN zl%ymk)0@BI8`ECMs%R&I65)M?n+2j|*EAd2MIxzpS6I%EFb#V4>i-u>a78E1HA+Xw zt|{XhY5EU2yCfInjGc6Y^GKST?$I$x*x{5rNfvpkF!1hS6L0O-K+T2PtgMWhBDG=V zT6+nJ!Or+Xi<0RH=ivD3SMNbcPZCquYtpN8oXs6oT9u+rsoLz$;yv6A{?)ZCMEFyx zd#ljC0kC1o`#p=PD0pH%Ve=N_%vMJ%J5lVAr`;e4yBer9TSi-!xe905FY%2|Z>~q8 z^Xm6S5bH1E6z;(P`ZWf_l_S*ofr#0~od%ufs~_cQh6A_=?(Ib=*c2g#>dk?kGw^6ifql zI7proot#=?aV7GA(DJ@3#k-STfX3cWNuEck5qZ%t@8nl3-3RwdbFZnz64$WH!gB0O zV`N$a%9KcgF(eC`E+{$^=ACDl8cJafS=1X*PWPSox?uA9HlES1J2^V0$7ULiR z6_K!~15-e5WiGhP90hz5=FwCp0>jES&x)oY^M52%ELgeH!5SyyYnSJ0=9f4ii6Y|% z_-&H})^G&d3X&!X3i#;J7T~HcmooD@SJ#NR34$QX1%~E95>4RRO^Go16Y*1nEDLV7 zUz2W67(ovUiSwXQ@!6~>R%Dki z8p#v>Y4a2wVddKuVK1U&+LxIf^ZpM%p1(Tl(f4$jo&`Gr{xn#*Ew^)@$@Ad@dfa<43o_b+W9asl zQF}DPA~^#jTkD}0Gd#Y87~nY#=pu(aFWk0|?K!C#?sj#Q7Ar?6E6_DIZcc;oWlt66 zxk3>S{TJq_8>5a0ef|w#_*TS07dUpdw*DUF5p60hcA6Iv19F-zv;ciZu658R^uG+{ zf@$*MB-1j?9=NWb5pPLPd!GTtsoeh;GaR#%s7KRyg*K}9|NgUQ&xX$ae|E4p-0c7R zcpjm*d;~uXmu8VI)Jyk8uqJ&OR=il;v zXDI+U2@8K)nCo@yV2uzf_Lwbh^#vV$gSMjt(!o{bG2SsCL=zVX|Hi-4ESjG!CX?tr z>hwGQl7M@hW2>Sw7~qxj)>F^l|Nd879!faFvG_+v);W9xf3sULPH6Yw*|TTQzTSU6 z+AO)$Q%iTM#S0qT{e96dM+mAema)ZaMWKyY#XMhh5ZZV-0(&ib~flrX(bcK`a+lcIgig;*8N89?3I^VWniTEx6yzU z3Y!R)f2^?EpSKDVaMB=uBhun3d*1XoQ%G9re*Z1symz3(Db-rPJE zB$-ei>E0)VqHX+pVZ&A6iaOFOzb{|+uZBUK&VpfQM|7kWKsk17a3MU`{0e6;$UF@) zj5>e*X4u;s^akBQ$2U5xy#hk^WR>IFX#T{V)a`I7+;_pgQ>)X@bxjB90gnl8UtbX3neJI0FeK)$`* za@47yL=^?M5D(`pn04SU3ov>r=7EU7(;0Ji60QturEh0=9M1i~#Dg7hVGbpCLl&mDE7PyM&$X z_aDRF@PO^TMzn{)@tF6J{D==uk&2cU@?9-W$9g5#;xmt~qWKX|nIFZN-lX`O`67oL z0!W%mZL zbpy61YZerxAY?#uUu_Me_M*9;uKaQI58StFLf~0>x>H z$2+~PlZhk)9Eeaf=X`2J40dY!4TH%uLtbFZJ;SAaI=5qvBm@~qPwRKJ4JYXTv0fT- z7G?MeXq~%m<-8eLLWezrC@CsDaTs#r5U$$Xe+EbdV|H!j76*J(A^E-Xzr_Rxq0Cjg6>-kO3h+5@KiI*q1Zxog{$?p zHG*2g*ZsG=<3Qu>HiGFPMg^G{`c{mmOFkrVHptL1x5IeOHpbplb2X3&s-Ekn)I_#g zy-3o6+O>Y&G^ry;b^oyaOi9c;hBUNp{r7fUTIZ5I^XPC~WR%(jwHaU|JK{Ji9pHN9 zwkvt7@2ig^gz;-PV4bcR-=+EJX%#x~Q6l4O%CoDsA3RZ?M`!J6;2nEyppGcWnB>c^ zfn{JUuJ4Gmb!2OqZOg~|bDeLq6NQm03}OjAGmR&kj-DAHsz{HFy>9|vZU6H=oXto^ zaiY%fwf4l%ux&5h9odx~wVK@43Z3M=6dawwDv1=~ut|g53bVpX!9A?kwAWB4!xPOTzH6aP%6Sfp@09lb-k7u1nk#8{)@H0}%_-)VX~K=dA+)Z#y0cX^_z_nH)7WRS<+hU%u-q=}Cl&>egK} zY50hKUl0mCo#8B^QWy8QFm^^smk@$kG?y80nTHT(_zKjWK4O8J*EVk-Rj75Dx<%MA zw&j`Zq${ODLk}lp*X)T~ht{oXGEqYeJp5a4r$x(&SgIe2`dx)XNBq%Q7H%moeBTqM z($)VYven%d?qThDvj`Cfs^ z_D%D;XK-zkV={{?PLIVrH=Lk~I%}UIi!a`3gu-&u)EHu$iK4~q5=+jL$!<2Brt&H) zoP9+myXWVL*D@(h0hSs&LC66S#!-tfx!oy`Z9uqOBot{_=8Kp{hHQ4O)EGEUq1kAN%hxRrw5$Pfw79)DUBl zq_7}}R&=tLs2U`=SJu4m%@(hE5(5g0>tMiOs2K*&H5{fyr3Y zA<`qv1_(?rchEL$qBRdRUp-~J?CWKA*-izG;R>r|NK6`zRwFN5*H%GO_J-@DDY~%+ z+gTMZ-OBbaJ)Ar`lFIBcUStpq)>z0_?MB(?o$%j*qr~#R(cTo=Ys2_#fUtcxj4~t+ zZBGs3TY|66Z~Nvqc?25Y5A2n2871LKjyh2S3Ftbyj>S8;CR-NPt(Jfo4*xoU&X^5b zOTa40hQumqPrZNXsLnMY{(8MIZm`6LnsJ3V0XqxuK8H{yIt`e#&uVpS_|3s3Y@S}T z%>q+cm94JgXKR5ia-6}cE?Z{Jr{CTq+%-x$@o=KD2R670XNjD~qKaHP^QD8V-O)5@ zgZMwn^z1J?J!hB}^1Ygo47p59_%asf!2aIj03g$4u6Z9&xc7{#t8AZJNmxI8@H(XN zibmp8_QA@dT3J!2G-dD21gG`YoHU~fKq38TJt#K^sDf2Y4q5`tzB!*0G{Z5(6G{Ub ziR}Rm5<0_~xcUlX)kThx>B@P!h+`ffCGQDo5{{sn44{F?1bq+zE>z)LG!N1oZC`%P z7U)fkcXko6>j)!Bv^^&omZ7m`IG&3Ny)BN%(U!>dW}w2}$rz`)(V_S4-qse$wrYX7 zU`zl_Sz-sUVFCf(mOfWmqc}28bjI+~n-u>z0VZ}Hr2pgt_6uZCoH#^S8XQEp zxqG3 z#3l0wyF#TryK%L8v8-@6DZka<-pbGvmGK`;<(c)PcN5H;M}(-3|FE}zxW8}5f7sg} z96sH|f4Gn5Lxm35*luPJC%sg9Jl`_}8@L*7@t{toCFqtJ(Mb``IzlW|)I&D>-(aFm5!V=)hySWp&sHe#TKYqoH}Mx=Je-n-p80 z=S!=^!T13=Md6Uu=iOs7f`%FcZ|n`$btQam-d0bw;EHP60HMLORr!1t@b-9CTre$s zY+%wi90{9sLgv#&x3@LM8Z~yOn7VI59ZCjc5q~l7VAIeyKvR?5)eNZ#paahc| z!L4Jnst76j*`<&!vOTTVR6Yw@H>@iA*)}tIXk*AKtUf;+%!sg!VHmYU=L&{A?T#>WM;8bXjL&p?O$Za-88to4jmP)HzcZb3U& zv;CbT&knd&J*a3<_h?572n{k?feHGs&H+$nS9K#%D7dyEDWcGJY>K$8GDz2W#2dQC zqxOrI)xN9MD$`z;41@UG9_R9w?y4tm0y>`I?2Kjs#nYRz9y!I<(KEN@_4H5&>JTTC z%7Nb&V-0Cdm|1-DmXmt8vl{=s1U^o>7Im81Ib4=FRV}FN89SM1nd8WREJ^kxC=Y2{ z4!4wqZ-lH^aC-Vh`|&fO)s3AI{je6OnIf&}mX*ahO;8!gmL@(1yw>DH)~y>8BS2ld zF%kLwgks9gj3;vwEs=B$p{||Ps`(9|G_xI2rc~a%R!HZyDkO4~PZxcFIATTTa(0K? zwKBKu%4BWmpC*#O-1;FJMM~kUqD=Ya))G1_Kjl2jWc8Hu6q8HL`8AHFGsylsP{YL9 zkh6)ixj7?uxkM@!N!>EJi-qzDmWsTETM7G0Bnwu=z14&ZW@x}AsGeDUJqKxNUOqWI zWoi4Cz`d-crah^&k}2Bo)ax{h<~hn*W^FW@7rVP09_TuMLtUxaqJyX_u~@C zdcZG>@q}b!{0cOXFF5o!Bow6A|E&eS<-nIkcOE5gaS+~|;W0@}F8|-~V9&|_ zx3~A~U~gmpc^^-u{in|A+5=IVG$DS>m%$Ih23-c_GAnogO=Rb66;l#%hZ!ejD7P!+ zi1;l^rGcS|SZ$XQ#T2h*RN8b1S2FA@160APt<%(1z;HwtD)hH4gIK{ZR*HL7X01zn zvkPulY@szRU37cP<0vu+AJORP0Zztl48_lsbg$)UPXDF7Q#;!Kr+WJTbnvu5 z{|CvD z9zG)KZM!&hCQ`bEQ82#5NqA;ARM`;|sgK@pl`A5*T$9a3?HJlb*&#Eagh}&8bi! z16-DK;!Zs}N3FR(YR&c4*5Lm;GGq5`xkR8Ejx|-b<^CeZ`B6h84cl#lQDkpxNsCbx zp)O)S`5H-M0pYR99C9@qnEh(yBco17cH1drhvjt3QW$ojxNagvkL?1MXOliROUaN^ zSytd%JuN9^4Yhv6G3RP3OBay&2&&@NH3^UB6fJ<^^5h~Egp$R4gfokToeC@GBD&g` z&V%C*tuQj$Ds9{u&kv=sc8b__H>+v0n#{(%G_hRZd^S=ZD0{-5GZIL$Sd1k~LB51&38IQRd9;r_GD{r^6mmG1wNn&IrNSQ1Ud1)kz< z1qTKQp9n!;<#|l@GHAt1t8@otw2+$e?)p2o{{k%#<`tTE>kGW+Dnb71C@&G?Hx`dP zc5iFl0(DyXrW;W_=clOP%c5QM7zSvO&?uH`3(mvh-LPEU0<=M#c@iQ4+2KhAPTo(;M+)&B1)Fk&+CkwGq7+cYl#Gm=+7{Z=@gx zfkAO)A;=>SDFg5d0?!_=Fraa5@5os%lqi5d#wFcpguPetMYWuJ z{(>*bJctsTLAP&W$h?h7%&Z%k4b^i$L(T4JXw`cfYJ6Aoj<9}8?rgPd#(YZeZs=|Z zl`{+_7e3VH(uXQ9e#mqAyPNJbkv&vwp{~29N#-tE8s}$PLtggL&!BP8G%jq;qD#d3 z_o|;tHNu#Wtn9RplK8-+1#GwK7E8lMijsmVqIlOg@aFfwl!Py%JX@sf9Qw;5oHn)t zsdiOhI-h2+3Uba?s0HueC4qyS^{yp)9(Wdlx#i;pzFn`a%Z=ZS%Q1rzMU7W0 zhCjRm3#AtHKTo?EiE(=apgQ{hY;fS({|pa@8~T4APgDB8ZMaF>v7XXA)ul{0qF6Qx z#xgUZ?^Bwm+@@Vbq>{v;VL}c_d(l57;hFH6=UGBJ`t*29Vr-p7&}^`vGm=IBf@`%- zX|g$2%FY#WG5;Xbbeuy(cq3UyeTu2?BI?UBWkE9HfXSuc=37C--te$DfZ+Q@Odv>i zkVfBSWRd1a=$}vcT6yw|72C@xqq8E(ayqJ@41NKiLK&L@phlRLhhU1M{0Qj%Kc6ge z3@>F(i08t?qp!G@)^Dxap8k&_$Q{w*7^vm{2YYV(=flB<{@=^0wq;{SGaZ z(=57*Vm!s+5t;;X&d!BLA_eZ4WLQKpK=dZXdCv^Fz|{*+R3ObK@(rG+^yX!hsTXNR z*b#w4^@dC5)z6SzC)Yt19-p4*N^Gs<6eslW(Bg|Yi016;jO6qOp8ry#%wC?HzOY}M zp1jmAvPE*7zss=@BRxW=ID;HtbcS_Q6GRCs$=#GQLZ}{~90fVL#&O&O0PF^Kgr2-* zV2_iVCm^VL9VLtRvVrqil#7*)f;@+K6Rfd>gbY}Q(IStMDLPKm<7`eclm(E4iq3)r zoxK8B6sIYi<4RO%_65zoR24{PL;e&&TkMx-Z{wJ&$>EgaC$1*(>zL!kPh@T z58fZ4r-z68hg%3ua?WhrXPX%g!XK0P<}D%g+bG67vyPnMhj^a1eVU*DOSyi-Q*-|B z4-5LA;nQaaoAduZo)2aY+zJL@2W=;$sqfb!$FJT;IgJt#f>|8w{jWHMnnW>1QO-`o zIKm;ip5X-cCy!kgk%WDn;dqXsB&R_f1A?ps&qFA5ttccohj96rHD}~VcRbX$OI+0J zUGn_Gc#BgZ86TVNn2^gwiaJ-r4(jlZ*Ee8Z5_U|ycXl9ObBV1X?8OXozUpKG!<{3V zE%1)Z`84vNHwSDDZekugrw*IXUOwy32)XPWp)|u?F}u6)W06j?AjHMOT^O*EvF$7^juKTe;U8zs7&dQ|X>O@7Li=Up5ppHDaq&jv2&3G2^gUpV^t9 zOrHE?#Du`T_)2{7bvR0uxX}l1EFg6_;LdVbjho?92fJ$JG-$$LA?jMIZKz2rjYBKQ zH4Cf{T0O49XUiphxlbGZkK4DeJ^sVto+tjp;l}^tUY<{&&a0F&mGdsJ`o46zJhvl^ zCvrcQ+qH=8_(tXc$MHn`ZIj`R}*K;jH;25&(o zR7A)8>NsGJqcE#=^Qvx1nn437*S@USr=%oYnQV-4!l?5?u9c2G-RD^_nM7kQ!LA=< zZ?(c520ReV1o2D7do>fWH|XIHBm`@B6;3`8FY0?GOM0yNgquJuh}5vN=W>Q-P3y9% zy^K&YCh-wEe{pJzVTReV!CChjkG2fEcGtq?n_+KnP-WpF&YDxz&WzI37o33i{OiHj zgF?3~WExRk9bK6%juv#^t*Ho&8$Q!Dy=-h}xn?WNx}nHMA#JxRqiUC%HG`>kF}=%qTu@I|8Ouk zaOHpd2OIg{{XFZS|2`R%a|r0=)fpp=`|Mb5=xIxpyeBj}H6WR{w9|gOe~NJZ~5TmZ%g))QDjiG`5q!r!?JbELAq@ zAS2pg4LcxfM^?TTgreFMEj0iu-xj-fjYXwTvpQj}l^4KjMMsc!e zezurQqW7rNFClNm@L^WW>;4?mEE?w>8N(@y#&0o@_9&Sek&qjXNdbb{iCQ~^T42WL z>}ZdN!W4Q~VPOa^RT-fcfpg;=LDSJcFD?R}W9#DL7b$WAfh3eTl6sSiC{&a?je{}R zoLXzKrs4KTn}l*{v>7R12640T<+n^$To}fy*4rlYFAXoAT6fZpFL@;oI3%K|=_4+6 z;_ZrXHmy#L>H|oHmtPI0f5MvLs*aWF6RH)i{BQm7pZA|Vb>qJr4EHwjzk7MsA^)og=)B(e%N$0X zX^s;``Tk-M7k0?{Vi0#51tE62%80vdJ&FFX!GzZpM%Y@34I=y+;f2eqHi$9oi|1SYQ4_SzKegW}WFm;E|HBTg0VXPl!qzb;+zYpz=+#V=`63%~$e&_&u96 zoaOy7$#6GJ@};~$>g<0G3h_VohliW^ulMqJ_&+w+FO$5$`pNgoj@bFGm%pvf&F|H@ zgJBsa`RXcL4G5v%$Vke1#oOg|xE6Z^Re4karF1n;?H@>jx2$8rV~y?1ixLA`l(V2b zbZ65k?iet8$aNVYmu}lM+YxBHm1SYSZE%0qV00vx6tDuR0WLd|O z^uLuAP!0V*I4H#bJ2-s0iT`vj&+_!&wh>t27`T7iAWhmGa-=I7e*&W&O%sywNS>w{ zNM1?N-AnJS$bdZtmQhj3MEmTCBo!^QDRKnLtA>u!=WzY`C(CLy+F3IB!PXfu zRiAH$rJOqEcrC92zpl11*sg-Y$WE~yyDF0%YGYDmt=Sf4R2^d@bs88$HQ<7p&5cy> zRv9hTRBfirFT=QrsZvc{fBOZsXa6e5`q?58v7Pe1eS@_Eor?6jXwj647pk-NX8$c_phmCHt_%!7IffxsA1Uink&H%Oi|Gw-0^>pxb zBmcRV$HV`#xjs&#Gnnbmk-ZPnh%eJSVeo}P#)D?=-4U?ZdwYH#9+TgmRj=w%kzsc@ z$+dAs&Qhx!=0>LRtsqqMnw6j$ah$Cnc!LnkC{1(AW>CLr#!*({;N>ezxQCX2)H+?4 zWppzpt&Y3AXZ~^>xEn0yoiLe|6T2j<*%)SFnK&nu=>H( z3D`KRhI_iWUW!oyDQ)oCWt9FMXVK)Q5vXD{4$>>5sT@OFQqxdI&@~QXErKcP3@TeL zQ6Ol&z^7)71>vkv|Cw1|37c!?%HKV8 zP{Q4%gSO)+r>NJWPfc-eG|Vu*J688BW&cJr_D6bFJpUV+gVmk?&xVEbfB1BNbN=7Q zv*z>vVcUj%5*|R`hwhUJSkjhki357o<9W5i`HMc98(86e5#82(+9{FgR5PfyAimNd zqZ*NF^TUR9YmE_&7;ROA!)u)=t-h6;!ixObdd~l9Jz&lG|8#$_@0|b74xerGfA{j- z&H4Ws+Xj~J$Um;FVD*?3l#8{?;sGcbOQqDc$ggTw*rbt`R)&r6ZfJAZ4%U5IAXc4~ z578pAH7~u)tTt98w}y0A#)HfBvayD`F{DbZIsUyiY+pOZ8uEogcD{Dlf`T0F?1(^w z63wFi{4P|BcQqJE6fZFpMRj9gSp2F?R>Uu@UY3p)Rj3tO*rnikmZkBt(paKZ+ih)e zmjt?Y%iN9qL~HDu+fUR^@CtSmtzd5qc<=scbN|09|L>>!Zv3D9;o(O9e=kqT{om*R z_kd*hYjN1+8Yy_)c&UkTJNYt*NwZ$Gh_7BH z!&@#GL+f?3f|PL+7@+af=KhE2IBe$eV?TBBzrAAo$EOE-oBQ8=JSF$PS9BbDFM#61 zgZA=a>vT=mC;`e4+psqx3dTe6-f`w+-3~hdzQ#w#E8;UN7He#rP9DP3n*J}L0IH+^ zhlj5GZ-0NV$^U&X&rFOTtEDBs z)mpvA#$UXRy}nutEsgWL5A|SMmj1U<0M*(54~7F*|MP6H$^UaN&noo)bGPyroB0o} zDKbW(pepjs_CwYc`S4wbvd9Cs*u_?-)e`EW%If#7FH$GrgHaf@WQ7_|P-D?=lgjs~ zNb+{SsV1qaa-HRT?ULmQUprvD-o~W8@s4U`pWPHh6WehgEN+Oet4YKB257id%TTBX zgfr@3A>K5@qfmX>!MMaXHX!k$5gLWcng{2<8w4`*Z&H6CvlY~(zGjxOwQmMzqayvB zRHVK+T|w7X2X3S6+B_9coBKa-081zUYxRHoh5P?-u(AKYm#5_Zf6AuXd$E^a9<&A! zR_dDW#}bxny3OA|5EY<1E9=$)a`?+A0j-)FE#O0Y+SC8`3cyv|KZtirgM*flZACOlUdpuas3oq%ta)~y%NZPe%n zKFBElszyb2fMKJW#k>H|z1oNd!7Ft)DnsK};$DYi<*C(w{kFH5GEBhsHYiC#~3mQUX=Gy6)( z{l?EqwA9fQTdvpBRMcsmX2ssPpRLAbz0XSLe?uKm&G~=$?CCRC|MP5<|K)z3^_>5o zyB^3S4S#T5kUJCweNgG-KV+RyDbnlE3zY$1LN`>3XAAvMJ&b$T5!pxYL)8;C;*r}- zT4Q}tV+i-CGb$gArh21V2z9PxP4bS^^Uo#8S4y$_jFPU@cix+9rDD0=Pm+}i64lCi zuN53y^|!1du(Zi#lz}aPZ4`o^lR~g`VpmWKHiFq!G1%(B{lfike{i_D|J}#qTf1zUc_3cvo+gSWgraSB9XrWUCrV%$bjktPYqc!DZ$%Y- z>G|;rXIT_t31bpOF+UyKRlg2$l;JtK!eI}c0`G|yIYz}r?x`FlImJQP>y&S<&&iFY z#r}^mT`K)U&HmpzI2^k6zXyjK{r~+uVlQq7Nr<-D);4!#jHx65JLap@ULzG zeuK~O?~5qIA?myc5(Y5Gw65+qz@c8xa8lO%H^9VqP{+6JB>p9JTS%j<5h(4bJx==o zO-sslb{>XME5`Irq*)FN&PGOkHAFfOR>Gynr6Ag3eObRytGlXHYdNV@u`@+^RO3;g zG#Jm`h?U9f+V76qjfx|X@o)qlEDU2&Gpu_qPwSFvdNbgSWi<`z)*HimfHW4XG^o22 z0(>c*M{`UTR7*XKmrYRdLUHi93&BaHRAOy%vKdK8Chwf}gJBexeTP?Ucyb!d)8~8CKpVl)+y3cBQt|L56`na=K}Jyntz}jS547=dcopSR57OosNCP@M zLVYQ|7zZ5C=7@NZN5n%Sw4dj3)owqPrg3VQ8>UqY>!Suuk_)xRxDBqY&`vWl!b@E5$0M6SB5z+yDaH)Eqp4PI1T8mW8WV( z1a3p%{&)mVZiu@hapf(nPUIx)^^!Rs)P{u46;JyTL~ba~hT>FEoE0T(EAQ8pvu%Dc zG9PQhZUj7=&3spzc_SuSkCX^FoG%Y&^pb1zPb(U(()pNf_q!fmfLUxJZ(0H+^|3FwRR=>)YhEspN96o z2?=pm{%m3T4^N%_|NdSv{>#&i{qMazrS`vU#$SmEFE;}I!CR_@C9n^G#~j#)O{q=r zrykSK>8aWOGz+?9^cxh}MzkpFjZxXJ&1KM&idR^&LC^EW{fO>oWw#&x^h z0&}o~9*bSR*>j)uQ@Q`ASuhEbpg#uP+B`_RVUW*8B*?-xMnH}PNY zotF{WOs-$;S94PGfSTOUncM@g$0Ij*h-Ty>HoH zz3*Z&LLI%zy2IY^p#N3xx11y$0mIaaH%HTIa`Z~(#Rz&d}E;uqu=HoP=vt174 z1xYAQC?KJs0&)rsu%MaQ-TP|i=br(`#~ltFbsm4% zy9JE9$UzYy2gxnCK^krp%Izo3qJ&OR=il<~zvZ3UkO0~X=<$w={zpW!MKVTD_u97J}6}Xp!(kFaTz7B2J(7Wg{*TtMj?G)p)8%&Ji^yI5E8_dVTjwyF5 z&&#TI_6K>V<%TFA_DV-5K}PK3Q-y(*}Z!n#a=4(b*(SQ-6(Ohi^PA#2H zVw7#M(5S6(l`Xki+m;~8)s2;`z@k#g+=QK*plZ-zV#q_v$4Z8NO#fGf_ z-m@T@7@=bH^kv(i{2gc*r919&qdCrSz;=b)DQ|HaN8=!Oju|cASR^!x8EevnLT3Wn zPB|6r+Id21sA;3apOe$BXxwSQq;08%MIEIwx;f4a_CUZlzIREAvw-68==tFJz(cGa zFG;Oxh!ZHSN{LJp+E6c7xxbCLDiV!$;>wLQ*ow6#OqG0fj59h9l3;2C%yxFuu~Ej_ zM?vvJ8@|M9ds!yM2C^*2QVwO^?2FaXY7C48syPc|RWpCA$i{g1jvxtR{2d?nH_%83 ziz#1jV#?u}+b^1@F~pjHEwo%qHjLIgO^04$19D~_bd?t;f4#M3%=@yQM)E&bA7jpP zmt3(;gRjfQfz|TA!)FHr*Z$|}{^3Ude=iSP`u(rAzQYOUhR`GKKe;%7WteOoUHgeM)(fDMk4E~x7MFLaV1l7%=! zl*o^C7UgIX#TZSZET<^LL5L&bDOi>&qmq%N&tGt_T-^OGCbRH?~Z3d zM)N++Fz(NTDCvtA-Xp0$3&xjQU-coe%m=iWQ_&BUN!Cx+RpL(3TH3KJPk68I)DC#5s2QPJLq_N zR4o4=&wI~%&%4k4rJHJ39?06P~KQIalb-Z?`5+)}T9 z*)>ZD8_$X|#?t5<8Q|e#9Ocxm3f0xGTLTadBg(dLN8F)}Ptyd`ldyAyl0_Vw@3SDC zof8t%DD51fq48-Jg<$%TkvR82j6BfTkxfVj)MFJiAl!iseY!V(DKdUO?e4IroTcE7gc#&mn-{hDgGT}QWM>!=K4pEdKI>Xjgz6$gt zier1(d5e6P1!05}wh^j^VVXtZDOq7+U`_8;6o~N@Ct+ci7Rm*2G)<24bN_DiSGk%t zEGZ_SVd;7BzG=xQX;?CgrZajLg-!dG&&YMtqEU!{ib7cPHg)k9r-TM^L(o@nN_A`j zWAUk4AaRrc1ez_XwizpSEjm%PwmtOHeSWE^&c^4ILp z1F>|yV|1k5zpfkG>9}Ltwko!5blkCR+qP}nM#o9Vwte!x>wnHVd+)W!sHzY3e5o<2 z>KXU_yRJEB$m*~b!wGDjL&TmxTr`@X@?iOxy$AD%PP7MooNqcM(qfRBpWtg(>m3s_FxY$!S z;5ln#Xhkk3Eq$r+mFb}sl9{PK6>IO;JIjdp-d$Zu6az7Owze4ru{+SUYFT@-COhjz z$z)lK-C=_}FcTS3$1PZC%U2Ytw^ryf*j9Yl3LJmAw0Sg}a>@*uET^YE(T#h%RBUl| zzt3lt23Ab`^4}`-wM-10Q~5#X-|AWYJz|H3bUiAk>p|QS6HV@xw`#=HyVcqJ+nWKa zI73!uYT*1gcoC|cBNr#Kl}m&=nXul5;!=sVk-g8O6aBWBQbH)c-OnG}j@gPuvy&t3 zlhijtd0 zL$S{STI-N)o+Ur?f^eL0zxL;&G!m7Vru<)qzBk~>Q;2MV>^hop@sUL;?>Wg z3w{p~qA*(u8YN2hL?KRH8Rf~?Qxl}nM)k^1E~Ss?wvP|?nR64)*WHh`r?^IQf`LPs zOA9VSkMW?r0Dw?ewIT*pp1l%)XpK{=QUhtln&HphOe?z8=y+L8Hk0RiaScB5gK8f_ zANG5R+p-(%C+J|=JD=6pB*AxPsdT&NljPcB6BQ|@NK+l8wC6tD<_SKp6dISh@mNvf znzmp14N|yFxdqsOLyj4jlWO|KO%IBl+d$=3#iPm96u*feVX|Kmk6I8|aPr>Wy9#ef zT!n*6e60xmaV)S!2T=(ZsGSW6fdS{;(`0%RoDA?&iO+VnESwuRZ^2uWJUK&;4V7{J zxeO)wAi1F6CM7wI!5C$cJJzHQYxPhB6{=^4H+pp3P%-e8lm@_zXxb-J>QM(@*J@t* z#c+Up_DOa_r}Gf6+l-B%T7lQNBJX)F2}5h|IjwLVG0x6yv&k*1y}ewwZ6UBZVuMOy z^|*G_z3gw{6~AT^Lr&rUZuBg4fxOLCM6h7)39YFNS)0m7O7=Km z#TBwOv1|9&tLw^#H}h=^&wU%=qvi?nbDnQU$86@!8FP*02*ZV@fU}#SW!nPtO~K<- zY%1nv%kQgcX1D2X^3B=j?_;}T-g|^^f7=4d{zDr@Oy#UYbV>#RcWI1w;!Z_WJuE{m zCA5QeVu#{w0C4BVz9$9W#DR}E@vz5Er3TE(_&cIUfLX}EfY9gifXwHUhuioq|0veh zzURumr)w%F{$w_O$s88BIN-5rhR!-*!y!W%{`>J7`Y}*w#xSVu5}Tbz^6tyN1)OOS zImDA=;Q06|2Iv=93^eo%G7@Y@EUn9&a3z=~UJ4iQ}XBIGwFL1Q>5fVZ3|6aTm zj{KAn56%y)P$rbZlEl!Ew9h1tA7tmnr?-J-lvH+)2L2Eh+-yvC;lC}bMgrAtuq8DB z;<}<1DPRvjVaJb&EM=kguWw3)Ts0A|_A4Rg8q= zU-&`1r_VER#4}^p^)i<_Nb)Ad(4G0c--5Ef@pA0#7$E43R+cN~F`FMB|M|za4&t%B zwb2~)kk#`+S-Wvxw%J6Ym~1t1XY7K)-8IR!+4#F-!DV4ih=e5Xu!_Wd#OdDi(FICZ zM3F%J)KKbnXVM1u)ECSzAmEOheSfrb3<}>_ZaO;d5AM%`Z1>2;$Cq3x8P%7Hxw9Qq zZM+x~*f&k4RX~ZVacdl!F>Pp~Z5Fq&!5iKX@aHI2fn5kRnv)}k{R+!`sc@hDkpdtZ zsc9}O9M1EqfU=v^TqHEA_F@}31~biq?2y-%b(l|h(Aj_N9)CVCn}vY%QnECD0$bXS zL4zfZHdfMt6HU!jft#?J$T~JHJ{ZIs{}&XzR@cB5{LOng2f7C4pF`LWvS4Al zQ4zA*MEf!6QtUjd)&2aU5R0KnY}gLECi3MuHUg=gG5KmzA-dnxc3M86u`lQ(_)Lip zL*|O}OTyFf2mr`*`nkU9&H$o*c)2`XaryJE+Y=CJ8P6G+#Jvr&50I?nd! znPKHa(y@52ixrx%xzh?gajols2Ud^D?c{3N5o48NJsr_+L4rq8`TqtPQJ4?~n3gR-@8_J+z(-V`6`61|jAr+#DwU2B2%Vqn}1 zl-hT2{y{3R?kpAkp-P60p7crx~cgMBfZU@{ifRoog8 z;uM2|T4Ii*7q1zMu=^+Ua1pNYUV};ybqny@xdn147pFm!Ls~AF$frLF(z*xwcw3~g zTDY)zd3d)_h~-?a0`(e;vdX3(RiGzgWOFaAQe_L%^0PIEwk=VV!JUQY5*x6@Fv^-) zrm?mhz3f;W=GQz>rq%A8e;WxJiyX% z$cT6^=IgTqad~9BvkQXUQ~DJy>zawC3qF{_u0H@(*U=9ZFY`KPKQP6*{cj+>M{hL8 zj&KrG=k>cg3;nxX9(Mb2I3%R!CkP}D$KMn#EW#k~BG(yGyztXOqJbSq(n630G1UGT zxmnzR^K7FJ$B7#_jz4{M-FALPUeF`%Sd8T@PPbk}=SC&#HZJB_yb5xSCq0jHdO445 zy6v7@LkWjE@9iFLWvWT}x<(itQys4Jyk|P_cr_6phdQ1W&OMh8Y+!Ua!LH)Jk1v1P zjGqmKj1MW?=j4eI!5~8$2J^xzDSk@PX7WIubVwkc=sGcQIW9cM-TqB;N`&T^f9^q$K2iy| z_+rt&7t>*1D;00vC;rl9Vp5hWh@6774l}oc`d%XMoKxt!@h}I82f#Up>)b@wrfD` z^^hG11WGckSPc}$?jXtOYlzI~YsjI>O&V-8UJ=&PpVDcEar?qtkU$>n={X+oT^qes@6UoF;orMblfz0>{V<4Ac?@ge+XUeTT@j5yx6i;zB$T*NKR=jRXiXLL zwD11?v47wfEz~?v#JD@kX9kYiBh80i!UIGkwZ3*3Jwx^2V27plDv@}{0~Va+_@JEK z1u+KD3O)z?2?Di{1qCnx5tsmNdoCDIB2X#Gc9{XVdY2991^Eg8;nSBJdmLf&{#gKB#3xe%jO>39yBSi zB*jw;Nv}hrqE(TTK=EXN(_h9)(56Ku2E=_j21KhGHv#7#r_-sB@4a0248$QW(Jpsl z0w7F-!8>Bz7a`kONHrzs0Z0u>>t-u|+T=b=ZetV~zH(9q44$72hzxzWxpr+H3lA7! z>V^?sh7zL5WARD(1Gj;leIcBIMmhxfgMWtT;y}DYGnXj>oBAp;F@P4zZTn*Nu1+zW zSUd9ky#kn&ydI+!6yV9T_DN^HrRvG4oaI8HrqZ<^8+JL-<#D4!mzp(GV)S zI)BUm#wuJ9<5I}Z(bVyF5a}m80JIO3sK9Tstg!+RuxYPk8@2@0xv0;E#8Xx#|A&J{ZTTP3&vT1d4^)BOYk9 zh({zb{q|QR2>%a8AX0}wAkyz0JE+xAgd6QiACwR&d`FK!h-QFAKtjQgpb6^t1!qu2 zWHWaLZ!&#@D&eC6Ck@%$0`FudKLkSEgaktKVUY|;Z<@Jr{`myEK|zTV!?5^J0MVNV z$~rsGQ{lY)xZQ@gQX_~*8zy=s%xs!*MYKrB7Sn!1$3cROx!&n6UMeSX^ED5toP*rSQ5W3xmF=bX~?^MQ+19rb&fV}2U4eG z6(0(-aQ|KA=yS;Mi;bp6qEnSr)UV=1zWJh8c=lT~yq|J<*m4BHK9YKkXVmFbonA!U zw2eE|UOPH6o^K*(N)hKkl4`DM%(qlSwnKG==vby|8y39iE*HuhtiNMAcyFwQ9iOCK zpG0WAohS?5PwXm6Zm(-|^D&JhK@*B72797U6sDEgN2tG~(YzN0Yqotth<+y&!MEaz zD?p0ykEjy|^p*YQ!aEy2X>(@7-nnTti)>VA9rt}IaFF*ZNGxAkYZt>Zp^Kia@ ze;)LQ`z*&ovu63KTfU=f;d2$XkZ%|WjBj8I*6F|njo(JBI@XO_57$%JYS#b&_ha1s z-oz%jC(=e1%i$wJc4WZi0@`Fj|Ht#-0&WI!Mw8RX>7;_@#P5M4PF?cA&8infTu;9K z0OraEm5)k(qOR)lARnxBF0Qk3_a3>%yG!)}T-bF-Y0u<(18S22YNleg1EF|_?Y^V{ zs}Mr3@C#_cMVmIVKVy2cE^PKZ0j{B(V=-2eUMz=4C`vPCqr@u$Mk6d)+!k`N5*HZ} z0sBe?Gc?{Am+GqeO3vG9C<_)RO6QC{13W@p=*7?_rnTDfnpQC>^`5KfJ;8B@!##`* zHB)aMIT&QBtqM}aDDk3Hll}QZ(z~k)7EnLQoA|)?7(W6NlTo_R)|&@ft7dvQTwMhw zU(mEnae;Dvh4B$v;0&6UV(=za?0J+ef9%J5Eoo_DNj*i9V zQyw#D?}QB6Yb+d!i5)Y)65C;>x$v&mzwpBT#uyERHok|y4f=|JienCbxbDLo?vs&@xd9W<)?4Ke<{)2Y_f3feQ04j0?qPbH*a z@~kXR5826)-snR-*P*WXW6tPLRkgv}>g%A~_QUr~*X3rS=nG z!swA;h?@st{#pdeney+yr-22ktvRamkt@4!%yHmWd?|rcrUCh_(a#c&+kXP=n$KRC zW7yE!ZjqSh2wZ;_Z`n}#yp&WeTXed1WO5>aC`Ck~II9AgEDb&;Q9kHy{6$n;k=~}5 zN?CEGn2MzyAikepf3E~Zkj3bZS`Yq8X^4`i(@8~RkB2~+@DV3ZzrKGa#veZTTcy!R zG|{N)c$lSJVZ3+gCXzLJV;q7Z+*?dTM@YO5bBz?6&5fi$1ZXPd9_K_bh}G!qbZBuY zyr@F9u^cYbGMl1ld!UY@00;-o+g;I0`PE3+jzrN3=qE4-h+U#05QmTlM09G1^>#ko zEOKsvy`v+p12m{F7!YMwHZ2j5Z(^4!qu9ays_KunMxQ)-`$ZEw+=v_3wAj$g-Z@g|nWE zCCIy1<;lCP=fiB5H^Y?ADwiTSA7HQ#m{jqQ$#A*Zr2HYjYhwN%YAv^91Tf^|jiT=h z1W6bns}P;hwK1P0*~EbUCG*u~Ua|O^!KY_iXtrmiadRT@E}F1EjHC_5BE_j|X@(UA z*B%a>Q%N|MXXZcJjkt7q#L&2svsb)RD&YR04iL%0->k@440>QcIJ61+5>9#B@$;rvd z&O9tI^IRHJ#SQP+FJ?C?wrEc{!F}OJv%gHc7JbNR6P+!Eoqo;GJ{}s^=pux0l~G*r zA6;qk^_LjfrM-CUR5K`FU=@4bjM_o}Qs$Dwd%#8gQZg<1(P^Vh$)8 z-kX}C=+5dzp(WO}5Gop8Dan#o&zG7iU2J@56k^xNw*m9gm$Z~EThwkPeWRHC)xepA z{kPfjTbB<4DQ-Sol#J5UYSuQe$(X8Ilx|Fptco($^U_a+j+qtz-mnY0ZQJrdg9L3` z(vYVCd|OgHyLwA2fni&z50SL!CjKl=*1t*}Rm&Nv(n^%h4ao`%S?Ct?CJ~^DOh6jX zpd>kvW>F_3V6TEF?V(Cp-F(oviXBs>ra>U4t6@M;v~`qVb*4fx8mFd8CSI1pMW*IX zrkSN=Z~5$wx0kBzRxBd<=bdG9?P#dKt)m{2mCr?uD~j9U`cf8?V?np(YIck3h_|}n zuh46)W_DPjX7x1ra(DK1#$e4KI!ksRkv%1!v-7-p3k1qlJiJ#eOFvfr$Cfw{0t_c8 z6_pV2>MBkGbzL?>>so8ERaGFQ!0clyeaz{@5fFB^2fw-8NaKnqh$N8;P59|P3m{d{ zT5R7X0y2mSzrg!84ft%XaIy=wRCxR4J0_e(X)o{zK`bnFgkF1w5^sxA*yI56rk^w$ zn^g-3=v2f|dx~uC?HkkhDipOCiNV!SG*}*vsym(C!!3(u(Jo)l8;#>`KMzWN@l=3v zChI|MzQPJ}M4s`*2@72-L)wo*J=>5Gx}NWGuzC0cqWm-S(Q0@6Guw^UMd-_d5KGK+ z-Bq(SOUx6Kh&HQ*S(+(Q;HPI@Uzu5Ub0c7au%;(R*Z9=k`j%hGO~nCJMt_k|dfhPN zRiKXyj47MndIf+eDu5GhKnts(EBZhMfPF!Lz=6imffGv}sX&Fs`1VeaC3m;MLqU*8 zf3Y$^p>PxRWX!5Fr*a5{JR1Gi8(dQ?mt z6nzA*e5d4^y7h^l7@&mfCdN`hR#H~q;`W~J1vas+3HA5a2R-0SXyck?u)>hZo!kpA z-W3VO3$1vSQ~|dnu2Z}KqBtWDst8$#AdVZ`p87f#`?{beD94Q;&+Br*t!(SsNS6D+ z#3XQbvH)BKs=58SV-@}fw+(WV*YLCodeXICxN-h7V2Rx2qGPE9GuN-02mx$CE*eRiKOa__ zA8k z`E!MhqZrt;mAXfXMMe|q6_vg!%4E-(SBmT;OBhdpC&b}&OR z5~49=?jhz^GUu!W1|Wd}Vo~a=nEdT59K1PNEaWmnq4N8acFd0%6@4GWMbRtcbn` z0#+#2(|$4)2!;TUn5X4me0~NpbPNT)SSE4D3X>G^+5T9hQt+lf7z$|E(lZn;;|^95 zIq_bv&k#p5&e|td6v+~00+3q@TeFRj*+~7icl4t??Qvs}YlOY{3B1LWI>>zve9ofL zPG_sK-K(8u(gnhLx3kFI9b(YQ6?l> z3li1km~h}%WbFg>xsi4Bxq;UuNMB^(EtS^C`ZqVrac}?J{j~ik5=HtHZFG$YD|R7|Uh?6|3&y0Cy?!)8jE#{lE@XvIuYz!R@GaZ$lCq2B@-VRT2FZ zN27|DMi7Z(?vCe5(-QF{n$0SXz2ur&i zO->zSzLre1Oxw2BR3|OhDJbu`@5o43eA)eb-A>wTVUcij+@v$as3k;IH#ZI zY514cU50*x*j^epWWnUxOt1W8oAe$gR1-K+k}Kt}bCj{6vsPF-e-qA1G2SCELct;- zaQca@p-d{@m2rg7=&M|~%?Xq}9)gr?Kl;n=F9?#*L-BN|ST(<{l<0+MIb#WJNf4>- zyNs8p)KU%Z1*s_Gi!^AJ>LVOD`CKXh_Zr=6fEB$zph-wfsB;T!0I6 z-ez(G{{)-FopS4y7dgjU~ z)VZV-Dsv(h#p*B0EH@XZD$=w(eI3-D6ilBzXghIF{mTx2%(s^u3H z+lg%3?8!Pk^3i+R6BRxkTO-(hkcvuJ$1);FJ14=Mo~7a)@OU~n?DH$#{XEUZ&nH2& zkduehp&Jz={mUpe?&|42KVs3sKR*JN6LpU|a2)k^`l6q^i^??0)+Q4b8i!-|?>K=y z5W`M1CB)iGvQzb^pU{N_Ia?@752~ECS@%nGWAmT08q-~Y=R@}CJ z(z%z@zc_}dh|Gq$cJzXJbko-MhN@63?HNYd)vsl@&Bpmp{FUa9>5UZq(Jvj0~Dl(l;yWb`lv`DA9#+ zKH+L}jyhw`m#kzXiClco%(C9Wz08P5m_tKS6W&LQ7sZS$P*{qyXJW-_{f47|ak=Dy zF_n2a{3eJhwK$130cSIL%A#bfO6D8FyW^fDE;KT*aM{HPqPM*XO`ZO6#ty0HByWih zU6ew&${J%Crh|XHixy5wWua$?5mPFGMe5D{BZE=l4JUn`eAjhHLP8lSd|Pw`Y8`^h zu@f9Q+kEc(kdTpuF|^+@N8WT>F|&Dcm0G(xT*(o<{)s(3Fh#;z)@9*;BhXL2TxRha zA*=jS+wFbaGEU#eTyVpe)+Z9vwuk1aKIOriVoV#p2M*6w7{{vIt$EGA{pfH#q{**p zJ@k=-g?~8w_LbMf586u_r)ySvtxZvI*qVPRof;N@>9_UTp-zj2eJSEakWL7IyuS^F z97IDe@#iBx1aFR(m3z)vd_{^Jxkpvhl)SgFT1VK!6Z_izy93O4q-K@obgf!U`Wjhk zjg*n6d#{YZ@($bGGn{Z;KTPb_a13qG0kD34X53v}>8RO`O&}Cinl+#xK?pmjFP@5t zo;AA{`^&&|<(@Bp?PItc<$-aLYqEePN18sRc@u$HzaqUsfqh!5;f;dw=ZDpJ&l{nX zkBz8TmL*_-uxZm*Vxx8-PwW$_R4o#7Y~J`;XqU4pPaP$oo&%`UH#;dzCbjniH|Scs zBu)a$ZgdLzCLANs2vBzxR02Jo$9ivK#;@q@3JCKeaB&A)yRp?vf(l`{0-PK~j*%a{7OPa<y~s6VCA{XRj`g7`@Ts>@WM@zI-5cd6z% z{EL(6S)fDS`QqjN1AkQ|VMmLXX&!*+P>j<+%1l1<2W8_FZRoCwQg7=PQ>rzekpE@B zaZOSvJ4r>_QtJAKoCjo6@^MF7&@Zuhm3oFZ&pGE!?!G$M_$WIOhq&kC+^zebUhVyG zJ#vbug-nKHgAuh$PWx2lD=OL}W=2#E52jC%*7Rve{a5J2=E8%1`%@S*bXG%U^a{GAe|ZrP52>gh`2S({Ac=ob0WyPFAal_k#519HPnP>I4S1O zR<791!s&yIA0>LrWCwx7`JHRMpNYXCRu&Pv^v{H<{gqagE|;sjlU>lT>VTZcX6>Lysci1k-a5>BFD8$9I36e(_P!C-_{8JnS%>%V5(c z?w5|(=Ctq}M%Ri>tLp(CWSIW!(Tk(}QP3dc9?veQHFke1emJU<*7$1K+swrmMgY{n zgMI|2DGO^T`qZrT6vhQpVv!Wp$QZ4(6*`)+!#*vOc!zsBVk$I-lpUMkoh$z>VPrZc zKeD-?JaOB#b{q}R$xGZHz8Ktwur(&=C|GzkpH1KS5YWgZl;kR!H0fQZ@AcB%@a7oJfCbHx&X?u2#?&K0gx03W>Dv ztWGfVG7mmK`4?~`5=Ds@O-?<8YN+D4nR!sDL+|z<9708{05MOt#VPrpci3WsW)l5_ z!*I|e?MBxgo6a4Bm9-y$fXfm&%b<7av-Ne>CUS0`CUW?a$EmyYwJiiW>zbWKUA~&K zi>vF!M;zIUrMt$yPi9Qy`_~Vbo-*$kE;OgUDt_7lRx+`(5K63Pjv0<@45tr`LV z`9DH8+wo9&RX!7!O5h=Xa^FwV!aAboXa{tGcOF6k&+lIgQjS!Ij!3cAPf0{0Y#k)& zFKwh^M-CuEkikfL-&96M0cq%yEdlAvc3HdPONg=(_kxmWHi|juTGR{1^G?q%W|((* zBH-IeWRTk*I(FG&q&LposRjbT4t&IulxOhzvb~u%;Hk`H5R;VOgnW|ocQERR0|U}Gh=7Jf=S8(N=EAZ6iV5LCecc44zk-6Nmn6+Q?9KbJ zR+H|dm@~f9Wbc;Dp$W`Df+%Nxphct^D&Rd?lwUy$xjPPdf|ar0(OA%#n(QG>o-b@! z4q`yU3Nq4jcjC0`NIg(Sf?!jk4U8_9ozakD0ctkMD*Yr~rJ7;jIA9Vc_MAbos7v7Y zMywhsj#wB-!M_nbdNLtKwlpkgT(Rw0o`;DrZ&rRBFOI?NFaRA2{2wiEg~cb#tU~*C z-?eZ~gpGBLgc-m5*=X@l=WZPpF{UnRosev~Zka8f^|Ga^*O&zWbrE(FS-VQchL<)d z(2({m3xuAavSc_7@Lyb;j%65$->g;o-Z)%F=xIGtc3Ps%NVYcYDpWL(UEMP%Mj?Q@ z9Mn%V82xx^o5V=jc|hq@nmVhy)oxbB%oh21Yjm4d6hVgE90j1aQu#P`p28 z=Q{OYIc&ItZI)I(QdMcY)fI-fY|KVUe0PPlV1|fTRkcrlD^PN*#`E{vp} zs~YMypTf{05p}>k+SXEJXaDVg#phM^Q}}RgfZ<3+ z<%1ai6P}X|e|~Wli*X9;cQbf{k;_9)6~TuO$u!IKW+{zpq?C$W#EiDqAV3uIt9k^) zt>NRi*KiX6z6aTCK|EAFoZPOi=0lpQhcg&P=r@ zeE>D&DuVFqk9Ac4(oK7r%7mWWE3181h#Mq_c>LAGS956@|KY_utXn9y*%d~aqv$nM z_{}X)U zzk_es06nssV`Qy$uJDLf>=rZ~?G`?Fwrq8%CD)U~n0Fw8m*FPL)qI`yIdb9qhe|8M zx`mVqm|mm!S5Ls>H&%lmTujoMSpOkPzU@SzYPcQz6U8UPe@(1m71;ULdp@jV1@4CY zdMi{*4r5m5z@?ukCPLiM5b&;9xkC&&W|c3`M3_u&KTe00t8=MBsc`K~vn(#Z{h0`^(xCbL5{$+HaxuPBGO4`?W`k7=Cj>ruYfC5@S+ zocB%s}(TaksG72v`9+W&UY5l7R7#hIu*ezyusKOo3Apc za9Qaf+>$Kf<+MNlQ;DQiw^;fGTIPHY3%(2z7o?N2mT)#|PXg;ZAzwYzma{_G=EblW z*#=q_tjGR4>({pe$$KCH*(GjZyY;UE3H^OO#J?%H|GALbg7H+|#VX@CG8B%XO?5er zy}=(<=ifA(@n31!HR`q&D>2IAU)wY2P?;K@Vn-bnw+B`YJElxYLH zCfgWY)i^eT0SV(6ChXBgfiJ-bhX_tvd2*dvyYM9|_~~FcgApIG4k=@msutUtL{w*; zqePPh?)eS$k39zWCh!(H$n5U~PM~K*7-f`bd}Y8dW@?h?Z^oV$a?-oZG*N|zT4JiX zPAY98d+>iq=Q8colhkmKOnNY9O_lJksHNv{DXPF7-Zp&Sv7?NAi%iFa}^o0pxJ{6L*qN`T)o zTQM3O#Z}0GKKom^v-K@!qO7cRY_HxO__+IMIH|g?Iak9bAuJs(=D`mbP_j$cS*J#| z61JCcL&&kbu6EaDalhRyr=}0OUfpP7Cn}75zN3aUkF?|zb9iRoyl`i%A56{guRo(J zF>he z>ur;G5JF=OQCCrrU5{L%v@|xp8Ddp_ArZTl{)2*J5MK~v(V0X5I9&GlH2*leF&8}7 zTwu;OUv&Q=f?auXLDl!T_?eTW$Q>#jY9XpLkK5u4huKS0K!w<1HgATl6WH56V_Ll4-Hs+S$#mJW0Y zbGY`a_e0a{zIj^ud+)dvUfUk{BryxlDHjAHkaKVMl`cCqWtTz)(%2j}``f>$Beb?* zb$$QfJnf}$ohyk+6x0|Tvcdt>rCqQD5iH775=l)z2}AOqZ#?=l?Mn}NjN!fGvn4+9 zgiU)1x9CEMXIAUK+ajXJD==+OV#8~*#4|A~xYsVT+!CIH>M44wCEH5)z0?pmTdbb^hz>WSmV2ZF>y!+h5K^w=HdOYQmXT;*AwH1o z_K1fUsAE6JubP(!PYKtH{0^~Dj-KQ9DFU=WJt=`k0yKyt#Ie{GuI03_0Gpi7!2ZSTWxRXzH3AwrpHAh&_}RdIa3c$RnHUGMLHP^Z&TM&ST5&p^ zGsMN7ZG;Wy5U8NN5ncukh8J~8Oaxyj`lX&pdT`}Baa?S`m^@DTM zOWU0oW|0y63#8407h}7Xe0@K{)m%)GZVj=3J-d88hr8cGs8q5mIjB z$)2oJcc60YJKXQO*!BpL@5QEIsb^1L7Vw&S$t1|0FMm3{7dbWO=`f+WIWn2xTXdlJ zH6{{+EGWi##z;&36UBCq4sR->)<8QHc0%sk3ImlGWj8|jWJuO zh&nsLCVZArgGyAvGM1` z;>5+jPgab$%Fs(8u=aY5m+~=0?MBT!L%b-C;FP=2(a1}x=}09k1?FP+Gvt|LkhE3c zUOY;MA$~9|X$&m~Nl8`AA}7hQKJLEBmvQM?xE3!mgnoQ4o51El6}99sy-*dn&~X2L z<{!v+?Ad2^B3vQI=#cm3Ti_?c%Q6f5VM&*Me77d?KDFt!uE(%NggVP&@T|pkH<#f- z0`5+`8(%@?^xqfX1=bkPDTL5otq4kNbwz zdhQ-*gqIz^(kdQVZtGW89k`|JjiyaXdkuLeM{Y&LzR=p$m^IhR0g8~7sb*`3i1(OZ zZq@FYlHpEUbaj@o&pys{(b4ng4<$cJz(4dy64ZkoEBYV(5{9{Yv+-Z>Q_KEe;0FVq zGhD)p>_^P5+qk}A7;AK#oWx?5=t?5kyf`D>F6_R{q`$y_{Z zB|5k?km7pG>7J>Mm9D2^s_9U}SuEF+4c4QN!$0_6+p=5b*tb9PS*++jjc&=dRiX6S zxI<)9u9$&UK2*i2h#n>LKV8pq2r)dHrnBSu;U?j`>?s#J2Dvo(8^2X?j4l!lOj2pq zjqFxug?Er${%#@frj0)sPrJSh(|kWwM0j1;VdFa3F%T>s<}0pHF4P^lH)ormlnCob zA5T0-1a^_5aCg}t)>LC3E(u`7f`8>2=7oQ%V>@+{ixm9!EX(_yn)rHn|8037Tz5yX ztY-KP@l*jrJ(nwIy-mUpCV7-)yAmdfQv= zET?;2P^ux0Jf4;`ukXeUMbk-BKU>AQ_(Npuo;$ zNH@(tMel?1UpnmiT+N$jg&hf!6Z3tMN;2B|}YGuW;T& zmsBzFk^aTYL3S9ut=;VO|89#}|F%V&usWu8*C0dAn6;FZAz}+fK-l)D83WOO>SD%V ze*%h+5>fF9rolSW>8pw4`^t@@y{Ps=TaGQ4*zqj+AKDu_+=(GP!SQXqp!&S3W@XW9 z(TMvmZga%7fIkA#I&EiNQ6AhGxcVu>7j&~#&UV7qdmy5OIRBv`6pr6Cq_oFj1{;z$ ziT+o~{f@`~)(mG2-~h2ing80C2)MHo#5R4?v~=KJ2uVU(VkPA7au!}t!$z;rl(eMo zjLp^|2Gf@qkcfz(oIQpUl>seGv~y-6*51cn^zKw4c;E8ntkL2BB46JBZ}R2-{~=#4 z>G%JKi1?T!V9AmEOGE~+%KrMAmkyY|NQh%P7)X6=BCftTP@kS?qfJa6otjqumuOkn zvM&Quh5lU(DJ+`CV>L*`3caHSuL<&U+l}tT*}T1HjRO&vj24#~jTSGSV&YU`LQq3L zxdxuU0(3J5jf$L0Mwe+@wo@1LBRh6Yx?N$Jy8 z0&=DHE)v6JFkRq|E==RzF1ehHO^2CaGPo3G-c~(ml!*-QZcf7fTs=X2<3O4Od&U@l zPNX$2DJleL3j)@-wV#D%h)?ukt~lDvdtDmzMA#SPgk)sibq4o&R^iZEeHt#QhuqVeIJ zzi(9hLsS61x1%%hNHlmi8u5yxu=o4G*%8E)r+6tW{%XpAF42WduzArtwRs%6V-(4A zWNpB+$6scyZaE3i!^E{DUP-IuQDMq5u}A^(uX5y}lH--d3XEz)WUG`Ao6$m4j3jT- zA1YbbFlTv!P3u49@C5eWgyY}&S5cvioJ6}slE(_EQu zjOjMeJSwP#IxPk*r)W#{lw#Hr>~Wcviu?U+G+lqYBq7L?-C|Inwjo>uGbuc337hQx zObmO@5J+25HR(sJ91zKaaw!*&xDF|U&n*@rq~(0JO7w3_a@Aw;cjL?VyddifV5!!d8y(7r3}bFTY1T< zjIL9$=C7nvUeYND^%C@`(`kY7>Y7LZO-(?2g%6QoLr^@~4*Z;Zr3lMIc?y(*Oq2yF ze6CzKS6FCW`TxV&J4Q$Lwe7wY+qP}nwrxA<*z9!NvD2|_vxAOp+h)f(^?#n{ynFBW z-RH}xF)~JKRn{7{s#fNj^SZC!byp-VluMFUh&E8eiWhWu5X$I=7n!Ttr4Z>PWhZH& z-QrjV3oQ_Q$Ill75?lnqln3R{*GNq!73Nt6h6e-VckVKlRyq4$+h`_1O<{E*I5$7m~~ zwpXdllgJjuT!f&vy}-KSzyvR;53)F-1c7=iX;PTZ7tkJX~duyNaUmAcgQu-K$Jx)dUh7j(N$m_dFZ~d;_|(DOpr6g!Z~<&j(Zv37%kQRplya6=wYWTasaJI zfV7X=lG2w`D7Py^E#}Z%USj9)cV{XsspZVgv-8zz#8_{VFH6AG)1}sK$9AM+(tJ^$6D{!7Z~|Ve@%$7h zD8OOn=O`phZuEF0=P!cf;T|BKyH;tU()rlQX8RE$5G6%vBSFEB$Ff-%RS3H?Ta|N4 zwn(>$a9Y-dxfZ0>pq}l;gKye+P!?W_Dj#XF_aa-$p5sv|p}q)g==)1{Eu?O(sBUai zr={&q^kY-QmTs9ozYA+9Q4Y!~+v0nYsMD8h1-fO#9%_FnS;6b^hbI?BbI6MVHB#5) zvg%`?)6_@mmDgZIbggEWnC_R>dCmjY+1FEt^4DMIn#9-M44hcGua@;}7LLeVlQ4Xu z_bWb5^j-fF2`{w%c$^x0p^qp6`l+m4o3e2@ckgIY$9!@p8xg|<=ia==)YNSUWe+V) z$JVY1$Njqwg|V}-=y6MaCMXDomZ090Womy=Ts}fmwGc)zlzX5L?30O?8tNY@HQ4Uv zgO2xd;rOARHmd1KNAE)qFQ03L3ex3bBkAZGd$Ul5dFb!u!}l@Rjj8c<+s)5WD$7%T zjaBS8h&bBo^m%yaweSmf@-uhx6L<2%OJN2X5PD@3B{vwK-!;C=ExByozse2N&<#}I z6QqS}M^CL@x7;U)G$cJzfGplA+nOM|Ke1sAQf~8j)nVpNLcpYpXB;3AO*0<7^N)no z%efjFi?nyiH-f5wjpt}@Ab`5!l8s?&oi?n$)^;Cv(p?=H;edKELAKNig_Hi>d zimc%0s>gS#2r>q&rJG;lz)Zk$dR-;dLagg=aGwQp#$8cOeGaDv;qP-Fd|#k)cG zSQe0w%+gmCks?ouD2!Wr;iMa264=3{T^~{3h8PnB7fsZDk1qR`8-{0&Rq%Q|$0!d; z;?38;;1SAsA{ssEs~^t_yqYX<7qBbEQ_fWq@t4dKbn1QW znvc4Q;=Rg^Rp7LhJYw>q$Kdi?Wm9#=8pdSTpZV5Wo0hj55|-87ZiCROeN@A)x@Rwe z@E4z59~}3p$F42f=&_Bq2H1q^F!ih@tTEH)4O;ecu{TxK+}DjO}C(C9Kyx z!NPW8F+2D&3{~q(lx&O;LFObFBRzdq4^-OJ&h3Fp9JN0eAP-1ifn`={iitk6u7!|C z_X3##&x@22L+kKm(Wb|3x9x%nyfD7?%Xa5)HR|ouiwh{SXp!S^$^DcGEA*9|!=2*5 zI58uX_yCH>*p6G;PKhV2QnNL1Ae*L&I{o%WXqORC2AfkNMi$x-aK(@zI>%*Cf1T7N z_Nde{I(Rp$N?gCIc8bbOT|y#sj*r9P@b83wnuY@A||Cj#HD3effMm9Mez}tORxR z#;ZOb;PiY>2-+Uk_4b!}GSymZFPX-ecW1{!fOG@6^v&Wyf^B#ETzG*^7R@P@W2A(z z6$vLjlJ8VLLuTd0?+)X~Hy4sS2K-d~ky6ZE@k3zsp8bZ7q7*+)Ixn2^Qlg++<>@fO zwyUdBbAsY5KEcY0m5NgxtCdW4!Cx#X=R#DHb1mCC6q=(B>-$SZh4R$R;RN@rh(54?ggLOxW)-cuKX04iEbvkkYOQ2PgpU)=&Jt@u( zSZDSVfdKtC`fr!CGP+qlHSo(ajsnNTeJ5k=55V+K-m* zJS~1s=nLdweE6YUd*QD{=X4ML6Ha-yT1T+!(K5j^n#Y|f7_IdXq!Ei7%6I#Zos`?# zoL`*il^1Gb{WKvyP|DZF>%K4GXRji=?z?+PTK3$SnY=`BL(8Ru{rCPIzPw+Nn^JUB zeOr@V5Ccj4->Jj02{yzjY2o~h2VKN=?2b9?hl&L0>NdnOR9l&cf#w0W_bpfY{t|Lz zd3LLJ&;)goI0={85Lt&!c=)?DdbL-1{ynt9e?Z=FiEpS4O2(|ZbApD8vM^1fHcsBV zk@0s@!d<>o7ZzeD3Gaz9lDEq|an^9X@*`9#XkTF+LcT}gzIseAMdFl}g5~nh^z>kC zm$q+}ds!6pyJ&24z7nd;1FXCJsrO^!yU1B3zm(2M=AeaDvl_Iqj_g@u25(JvFx?|p zS!B^83Obad*&qspEGDk^pqK_ynu;6p$EVN9p3_OAhaccRCrJJjPf+O$uZajQoLzE2S9aF)oVU8RZlH6(Z1?O z*g*Njz@7*Z<*Stn)yO63`yZLxFw4WQO{RA}x-D*%XmkjQ3ccr6OKJ?Vhth%Ft9*>=bW+xE6`L-31 z3T*tmtVvAqtU39->fQfsgH8RHd%c+|+pCl^fQqm>HUUxXAFOt2vVHqbn*8ED)e%yV z%n(@GFS#M8sHiCICj6?csCfB$Qh#jny%RmpepKS9UOx9UD%+gD8vVcFwLJgEYf(vQ z!G45*`_uf|Whx{pjcNasDBa)fd z4Zb-a{HdC~5_EybI7hwWXGl+D!amNR*n~`gx!PGF3kMAD|6jVc74(0jYel=@M{C-% zsWeAshv?A9X%>1083|RASY_XzgubIhMy_qoBnzUrz$wYXn^Fg05s=yAoy^=~f|J$X zj*$Q9|ES+TJ&0;6aTUVK{2#gxXupi*>e=wvzqs?!r4@RL(VFPPu?ZPe=GHJ1qwho zZ$JU)N?&7K{;Yt3WN4{<-StfgayFT7>K|REmEZ`0G$H?-x4GRy83u|r?H>a-diw3L zVvP4NE3SJ+JL&sSdu5A~GhL{S=`vGuRhR)euz)ekr~Y>y6jNNu* zn&zVh*F?H~rdmkhn!uR2M%t>CT0}2k7p!<@%AlOaqaU57pME1n5isOM)m!J??ZMTy z1928ca(1SNbXyBV1DN)OglMB3m45G^9~MC4N>XxZ+If?ykK@r;C*RJ?Mq0_$t4^Vn z|0*9ojJuNe&+ZZY1>qqPut z)G_*~trNKsE5hkWnLNv3%f_>-NKNb4HH3AJA{Y!4S)`^dh3!y>&1kijQCVS4hz=@A%>ZM95>6k)YR3>1oQ_zQg?|IV>FMf8xL2P) zh?4ti$QE`npq|yZ_2CxR`iM;D-P|T7i=5%XxNgIG6E%}lt^?I$leD^D%T63`RCq9Z zA!asf@;e`}SLB8ps2lqUpp>^rIr8{tA*gE;v6T%eIP$u1@;}_$?h0C1ir`>$ z50cKkx=4ar1prRh&wgUyl*FxbaPfMvxbb?x9a#x5;_-;NgdKsOlp{n( z_1&bzh@Ec;bKxgoVx1}-DnA|Y7jOl#QEA=a00#T*T4$hJEyx$}7xAl$?b++u#{TBG^z&7%kz&2EuR2evs5S z$}YPoa(U)BxgfGzw#oYZhU`jeE>U}+tDJcU$m3J_E{KFi6TuBvkOO_gNyzqRu?~aB z@L@sT(gG9GIQE2_g*>R+>}>|1m5e#`z^)hJUsIMgxi7t+blki34<_4cZzaJ#7}*YG zj;IZ3ppl`^*Bo(bW_hgU4MQbHs4hjVDcg}4saEAJmCrl1%dp}?8&Nn-k7BOC* z*5SrGZtO&~vDhT}J4b!d#94~NL@r>%&&MkCdFihS3YV}>cdxLS>M~`MbL`77m_bH) zIQ_SDE&u#uf%tBDPQ7et4pw$*fmB|Y1hEZd?CR<77}UI~^t~6@6LvRNBX~xtd8Bs5 z0-l~7?cJu{+5*=!rl;J^{5(87IEk_CR+xRLIuzZ0qEBfmT#|pa4H??ih<1&Yp?#r! zCa;8}o+I~Wv;;i5a4$sKAU21Mlk+-W*VfuGq+}k&A5;wJkKQ$c*@<`YcJDZQ(O~*i zJ-#{(cd`(`M{A1qU_?LFjEWy>G71Xe#i^U+Aw<-mZV=@@`BLe&N;Ro=O|?{4LvgMg`In}H@m>tNjx`f=*pk zJ5up&a8#uI(7?6o%y_oafA&a}@l?X++ifyBIR5G<<8M&Ktj=W@lDyY}Vw7&nOHDefZ#O0_7oWZ!J&hkEPXN^*q@8Xo+qxmU0I=jjrYX7SP0xS?>g#)1+u7!<#S zJ$$jDN_6csE_mR^Rfg}e?lOGCFnn3QVaD(ccCo?pcx4#5r`m;<7FU@LcANXgM!8mw_+aXc3L71JlIfG2-=;5 zOJTstb?94Yb3UW`6NxgGE!5t&XUX6MVo8OtLD>a_dw$M#!H*8*Im>Bp1gp!^X*V1( z(?`a%uqgYAxGw!XUUdrK6~Mx$O>ico1)qE!A1{=|0R~ZEUQ1ttUxuO~@+q%5iMSNu zlb8)TCawXzrZB)PfUkzKswh*!j)`OBwv$=%X;%~Un?WK&ZT;j~35*^v&;H?(VAV5&1kRd|cx zU~_Dg_`2m6@}1*&ImxxAOZEM?A!+J@+<%NSB2IM<;&zw?@M9ynGSHidM}0lcL`(_MiVMYb;l_R6G+z`LcH9Cm9 zOXBUC8PK~+^6Bq>omTNq!%DJ~;4N#Rn?TTGCR%thUdtdtQPclGdbyqZwKq=o&50;w zH4vCAVB#4rliuePf6}~p8o-KZR=pSuTSV$X?;1PON|+l9~C7UPdj61-Gc& z0oWnEJO&sMSGf~vDPy~{w+`qi#l(@6S4Y~f1!)lCfB(C?+uNwKX7yPMylpf$$$Zvb z7sjV_RBSX#J}9sv{Dqwyo(6iqQUcg^T!D7~=l7p_Oz(E%edKwrIFF14uD9bLR7bdr z!;w@$XdqOWgdp4~-%RBjptx@{z|TOG_3NSlj@TE1Mfgef%pmk)FAXeyf};Y7+=7-3AOlDQkpD;qkl&l@!IS@P zr?ZBZZg{@%7fsA?DFjg7hXN=iu!2TmO#w+M3!TuEKpZ{wtMw`taAk5U_Jp}O?&c%> z7<2}*2uB05cs$y|ZT{Ad-UF_ktN#Xjp2oe35bx`K4QMY_+6K6Ig0zFVY(pL}2Jis> z=oh{Q-NFON0Dt<)0L-AqF)~j8X2C4lry%SkyKNM+MzWtE?5Tedh0R{w06zh^e=*n@ zH#QkOok;(VTL^f$|HUcF1(Sw~H}nDzas!;Qj1mSt9DBfG8$TYbc~%)dToqJCG=tO~ z!+_L9GXgF!1_3K{<(?>Y(ME^>*5xw6AbbEY5Jkre93Tyd2ks3RvGQzUA|3(m4Ni0O z1R8;k2K+GqkkUnfh-<1|lSTl)hj~Mz$O)+H>#JbT?mXT!`4y2Dh(ZV-)Zpmcc>n=i+KueZUk?K)lH#+K(??)TfW zFmGa?+7a%c&oE4(=c&RWQOnAJ+c^S$P~RcbtYe4T=PRRYA^cBt#`SU@A$VJFS!P$$;Me zgCF_k71+ZOAD-DOfAT);_p;hF4CVY^kbPtB{(is@xKO|k39RJ(8e&LdeVOiPVXxgQ zNAEKs=J2%r&nu&yUs%Aokpp51CDiB@EF}~;Wq2kf6i#9A=q`%_V#1t$J)%36In}7j z#ji2q9>e00k>|4Z7qO(s!og1!j;l#{V!=+X<-*iE|1hvJ+ed<-#Ow)`tN+Uk>a>N? zGRDgz=`SU=l`O*dl|ycrYKd<7aEmL$Y75ty0Rle5dyYpwfz<=L8@NQKf(LKAjJ!-| z7n*LI3|iZf*W~w1yxXnxHt%utb~-IW4h3^=PY(ug;mfRCff`))gE^yq1lW&9cLv33 z8+w@<-lWWLy#(51dyA(M#`Ey}J`0Dc1mW?Vp^pvA=Z6SYSi9rk>u|4lA?|82U-COeF8=^JguvHfh9_NDp`FSYrJKP1+(hjZEacvrs$z-9Xq4Usm>mUz_b1dl{?v9FnfKnr*dYH>Cwq~n<9g)IEdtH|sxZM{ zY?>REG4tyrW9wVo+xJIoH=Iyq%BYV1FU(2yUzihWB$gbdzjC7N*a%fm*p_eXzdUbd z=j~Mt@zFeT@5mOohp4I2iEhFF2K`gfGcPD)Ends^+DMe7TOv>)q+ZB~xBQp9CBVFptDw#s@ZXETg@!mYWcb@>_6{~`Fj{8bN;6DZ|3Okv3r2(V@e+26 zS$l{3_;=u7ixNVCS*?R{Ag@cE)$_kX2T5dnO?k=eTOxte9>~neg>Wcw`P{tKD!fmd za4xwmd?K~`wW<-l>iTi&E-j2WnP@cHBGN%5_;8%y`PDzL()+pzad^fD*g&G&z10Eo7n`?=pB)Qf3Lz@tYwJXpNI z2jZy6|9{5{y0510haq`(DYn(N9tpYKyg(yO)h%f~HhxAtbW~t|T3DZ*R8^jR2+mG* znjv`7+t0Jy>Wn2-CsWmqq^~8z2~GJ!X(2Wcggwx-P=Tv-9%gD&1&VC1j@-iUksOf= zHPAfHre}ixEI(#DK(WuOH_MzNXr>w|dYT*(xhtQw!t7P>q^GN+>AaSYv}t~zc}m|l zAtWJyQ|mYlRZdCvy_#YJUC?&)?3dGKfJqb$2!9iu)%6TA*~P#97()Jw(R{ng4( zwytE*K=GR;{rEy$1A!vwjGc`0mKY0lKlBq`e?IkZnQd}%<(ARRw?1*;DgxDYF959rs1J(S0;815B zfxlZz(~S(5yvUs}1xW0d%M|!A{!K#_hC=@n0nm2!KlIK9qI;m>t$Kj^(o1&MKJ{io z1amjyzeuRc#E&OWp@YZi>bB`Ge`QwJ3Vr^;rC(p~oP3%QB4pzqLxEO}ykm3gZ6L6t zU%plH_WeC<$#AjUg`#mb`n^6AEceA`Y|}9AchIhs95J8w>$mux61r`BJf6EuNEy6V z&V@x483pBCWjeY87eYuc1?EVN`)z&N8?1Vg@@jGOrY5J;u6)8<@W@!7bFpgQom3$s z!D!$l(*9o1rhCEjaA2jbh_b`q|athLka3{&k{G15-^2T%PO zT1o{!)c4%R{i08G@yv48;=`kSWvmKiOj@lEngFGmKWbmx)d$HB+||qIX?D6*W9U6x zT8q@bewBHs{N<)mMi*k#3Zw2x3-kUuCh<6>f{cb+XBkoU{0`@cbrideyfSgjqUG&* zRsU0x#v_TgHf7amm`AJFrGzlIa?JV9Uu>3uKLhC-J0~LTK%(|vqGiI5SFbBtKk4k? zQ7IQV(ZAfsGvlxKmcuL|TK{H=@MG;SbbZtLrV9kcuj*I$F!9_Aj2bSInBmi)>ZebI zknZ!N4DcfzXncgva%W6Ser{vmRb~2rUU;QQS@`0yJUH!w<+^lxnshr&`}_qvXWkGB zuQ7u^zNH)e!wvK#i9n;U*3<2Nj!DzCpa!VNs)zz>zy+PZfypbOywv`zSQI8%|Is7= z_J%3>v98{$@q?bs)zgnv#lHzdUbeyWg5Wgqv5pjB`cI*r_$Gy)g zM~_EAKWGoSbiM8JnRfRKEx=Qe!0vV5fYG7QL5!V$Vbg?88lE)0ruy( z7d7{O^1zo83{=MF$YJ*6H`+$E7nghH#sjU9S4z2o8#B_G&4-tLvP+ zQ`VCpb_q_A#6FPNAJUgTb7Y41JkYxB$C3?6^ z;>em+EF@Vf6!u*=*?O0l-8JTifnnN16cP^j@8-~*D6DNi+ZIDpn}k8$ZI#I8MO=M> z$dq4?N%GAuD7hI)m0?tjVO&gxX-$z>&VEsfu<)=Uq-tkOd%H#PM$a4^hI~x*SWYB;&u;};*|{D-g|Q{zgs?MRtKqYc#fg>mvxjDJ#YOhuI6kHDR;maKYpmaPhzJkAH0 z&I7$LBQS-8RIfbG=W8{y@3ChuR$_V$24bi(z_OmN+~##Y8~G$lX@BG+zYv{AyH}n` zs)Rq@ofaS;04bX7neFh*KjFSvyGquazAwx%b0cdpex{Ayjt2UdVblRx(42qSFDsvdkzN!PK}Suw|m784|Hvgp-XeTRk*S9&BWp&Dsv< z%KDrcZHJ=7OH0d@)fvDNVMHsO+Jxnnxih9_QzH zD|B*Jgz~Az4d6+Ke-u)cK!sbzA~Imi*MAY**N1t7rA{iuOhf+6$~w6U@pp|bxj&V% zk|?yQ*!y{P{G+Guy`4@ogr1uw6He4EgQ>CdF*a$h!`Wf84R=2fGx{VgZSyRKrv`Sz!PaNRd@mwjHOGfgPM4xRj+)H?im~hV(^X^!`<9JWR`SmXwYli$ zP{RhqdE(MPk6V*o~FQxw+-!da3 z<=IX{pa+Zc;&1Zj*Zcx^tXxb!vS&J;rM@q$>S~$Mud(N^sW7YWCQJigo`M&i!Cy{z zQWT%PCvaddMDbsjL@!3C5~gdtlg^(M=@7v;NYGygVD_1pOeJWF&Yy=@$I8KqJSET* z_E9L#Ulu1vZ1h2+1X!$oK{}|f-@0kM%|;n$O;exXZCZB_CT{eZuIb{n2`9+<;xOS= zM%dfvQ5i{&>?hUgaz$*1P$utN5op)2BWhJsfuUNuz2e;=1a1W>lduLgEmoj*gIZ$I zoDDMeWl6MilTLjn&CxzFTKcZk>tv(f7+UPDDjsAVQZDmXcpZy>_Uv^tS4iJq2GGhh zbO%&OQ!d3-^6NedGz_k}7H35~&71beH6YZd8@O1FT3m^8BytyBs5D6!SkB^GXinba zQKrqyRkJqF#D-ZUYQPH^{*p3v7c}y)S^cbcqHA6-x~TU;1@O|L?yiCq$7PP;QK!T% z(d6_n=$YcpHCq%-$$1RGPe}!uSMKMcCKz3IxP&y^X{E0GWXqt?-1^4n>mvDP7m(Ha zNUA%XlR@Gu9Q~8eP~ypmrG#TuWrSmK+Hr}8i)9mh*Si<1Pe;huo5`eEgK0Lf zix{G0)&f`db9JWF@E9;te)rzvbWD6RSH<`>RM}LF-D#8s=*JbUu-Y zC{=lr2AIL5E5pQJ1$@TRJdxee%y`v`HjW!5^vL2gk6|PjP{!8(PjNeI1RLPz-gE z)bgE?Fe?u^T4U(fgB0|gw{a{hiD!0OFxWavh!s`(S}pY~^ex9xW94Y=tt(ixKS(Dl z_00<5J%lKuvTP}&7J^^wxq-U$_rCAHh+jM@jrM7DxL zxsIX;A-lfRcpo)HqZ~C?2Uj0N!ReLfT(p>}&GH@(U|2C8q zSR?ZEsFopgcem?sRPa%cCz_M-p2FFl=C)$S25@WoLL43q0VLsdt_fNC3WaPzczw^Hb_MZc4lX*mi8u zB%x}ZxF4UM85Y!7cJaA=u1jg85uUau=d{Z8E{(|WUjn>x?d$}x{SqkDoRC6J6;I9t zThn;#f-8wRWcO{K1%cBik{ibBqWs{$4Q+355uIWi0`m|zHw&a!hNWeAH#g*bHi$?{ zqa!^FH^OoOh(+m5W{+!v@SysG!IQF5l`e_s* z;simf7mcg$086rN?by|>?qNi2g`VeN?Mb0N+afHmJ#k`1@V&t@C1OSUj$%+}(<=r7 zpX^!7v6!a_mNFCm82#;_>zbz709BT&?0uRdIWP+-mtiVuh?$d1f?l#`A?3$n$!3$7 zjjD=HLxU7N^ix=xl+-8NeCettZ1wV<3U!fyGo!v6KkKQS_J*;i@4JR%6c7 zzXfB-qfF#kFuMW4qs0(H#9obJBu zIzL9C=@t_2pDcFaVOzZ}y`w+)v2-G%xOwKy44Dk>=6MzcGGI+rE$(d^1{r_E%=7FK zlv~y``!zjm@*Mto$szl&A#R|exmF%7sXE~3)vzN2J-JLIQuj9qg)NN@wF(_pI4Xzw zIcjqS8Tdj4N2_M_38HqrZauhqrAV`Gy|#9>W|5?FwdT2P^F1kwN|j~-Oyy$5qH(@c zII}+2Xhh(pt+5}%iqC%=vNLb$9Y2TEJ?> zbfvl-nv*w#fD_VFN>J;UA&{|RT#ay-9~0z&L%3n4Aa|N8)VaJMPGagst(7PtSQn z&rQ@EiP=C=MlgsM;Eo6GUO7t*Rx{qq*XOQs9dr&@zND+s<6W=Pn4P&ke)G}B+qA+L z0N^8V^Ug!;Soh5W_x|R_a*Ew&6Ev@%lz15Y70Eyec<^H zdr36v^mzr{0`_GnfLn%aV{T>J0zeKxM%J$~X?FPAiFf!{fh~8$vA-FXyP-L6&8TfOTkZx8fP{k~rpbaLDn(ynhgXVo#R_bT??XXG%sN9R zD|IEv)nQW5hYP(Q9zs#{ZQ{1iYuw#^2~rOe{M>L?{0y+qJX>jV;Jb9{(e|XY3A`U7 zumk??^*it?Gp1 zKJQ*HZ?pOkdtZ>$4oDA!w8cL+Md(S-2t3|`tp`6P$qulf)PLEk6I>bs)*za)PC;iF z^9)Glkel>iPD)0i@Brx1JC5@?5X?>ulO|?co@~ocrH#vi79(k3Mt$8lB7NNrVA|Po zp&*Z`vRo*SjM)xLxbJ$lonEptalL&Fon1*`kl3>Oa7@HKfZHGPSaA_d=rp*4SS>h4 zJ4r{VeCn6N*|6Gh>B*A(&GxXWMN*Z+ys$Dy1IALTzD5jdyR%Sc0`sbzQ_KJjL4T?K z#w6_R&(F8(-qwKEa(CD7pSN*z^IdnelMV#(G|tr;HF&1KGHPrF$A)FB8Ro}Nng{L5 zQwk7lnN8ucYPoyW21I(md-F`mqJVdOsETwDtWi9Eb(P=CQhV3hgI6iRT2445o`-)H z5x{Jox;0=xPL?if8MAP!%8Gj-z|ddMTT|GZP3y?{EBqFe-x#*>(OSn0o`D8%DLsy~ zyQ1%9BrD^F>*I8FSf@ke18}d9!JElrL=l_%-#4Y1FsC)W+qO}5KgF6eWaU)DO<1`m z*)p_ZTj9Rb7~n?hejmDM9P)$0Bx_gSLPndK29AEfJIA*ksDT81TGn679w>iB$OBh6 zReOR$IL%2T)<)`^ZF!*U%dNBTccTe$ekFn6RA8C$pX^-?z3jYSxJXlC?SJYVKmAyC zfiXJ1*Nv_I_(1He_PV^bFeSL@4ax0CuVV&@$&O*`+k0nv`fg^&1RV5eSz<6Qm&!{L z9$8l}L22t&{^#sr?4y~v*hxyJm0#i!$@l&4yUTf8q{s=>A$mq=N5|Pue0eJ-A!>qK zPpP4`?5ou)%Q@}F$Od37+#(VNBl9FxibamIjQh2U0-|LsyWPCMAtYHAjvj-}Aol{F z?##SG8#~$ME4g8gR&p}=ju_v<`|@*3@D(J>tavCvH`A=450c&Wi@k9USA3FPP4x|; zqCK!ehi1iyZIPe2RHvEV)yb$2B+Ih+Fm#~is_V!CwXE)d%CSW%C7ZFOOZc*q+8>>i z9wU@jc@^Ycs<>YnMwb>LC6BKPT!O)^(05^RSg{3EZ#5#ba(c@`f%bspSU3R#<< z<)(bWis|naei!S$Z+Kc zL}zXPa%p@cxV2fRzb3D<94|~yOzlf0N|p$HzBAVAM|7c_IP@T<-0=1gzfBH&3Jgbz zw^Pn1Gd%4EuO92fY#B2$jg7xf@O;NS5aYsjy|u2Ri;fn>)N5$(TacWUkP4oC*&4|( zi9^RObUoxH(FV9;%W z@sbfcbR1n^?c!GfZ;^yiyquCFn3S8!YC|n{x|;L;prggeKvt#tI@c(`a2X+Jxn1m- zs4_>L>oI(%$CtL9`5yZQ_F{9Y$bvQ3XtjBvc1%0=wH6~FoH3nNB95D3yq(828*`E! zFGVBSfvxtz)=yLRy~bhwmPW*>|3Yy=L@4|Uf}po${_r+x+DkVXON^iY+LOt!Z0%qV z_((L~7|7)0R?{eWN(|6wkDkFma%PM@x=&4y6j+}g7)s?Oa~yAC$X=^)RNw>si&9G_ zs85wM(BgBsMVw@Aw~A{cWIR98DcQPynS%s*@wMs5_3>pGICGsXaE`zjcHF&X%zPnq z{uDmoc#U)BxDWA6ICRx`l~kMJ-xh`+9OBs{_!_J|IFxKnjZHJNo!Wb|{mjHBOK@E= z?|kd;TQQ#@^dzuX(IOEkv0mZeVd}Tmke#Qtep$yO_AUn1i@iS%w}I!e7`>Yg|GH^- z!&JRYVD!*=_D$M{={1S)Dt%>+YOpYYv4zy2iv(?RE+f}s&C|_{Y`jD|+r&;b-mw`U ztnz^AL<(-TMp9q=@ESeAt7I{RZD8O@NRfQFv~Wp z#Od@QpN%YRRn=;x*f6{!)J?uhkHDGq{(#}n&|HSe z!cAJ4|4#z`C4}s=K|3f#73k)`8M2p|RmMkjmwjSq2LF({;(W7p1D6>R?8%(ml^R*c z2(uNW`g=iG3|krq1dbQttwbm#-dsqZTH;G?&x4+5JvvULq4I ztjn4igX_{(=bykS1xcElSY`lR6 z&dx_m>Yk_I>3W5|0-O+)VHT@M54WBg_r`GbV!P9HKU+EO7^A~{(hEtXGi9yruT-R+ z$=`e*F~jGg2Xr_5^cNNuf)3mJB(w*JkJI(!!B#@Vn+v52?!BvL?DW4pL#5);5_0<7 z&!iN*eMasx^dBgqi^=enfnLF~fi}0kKdopZnRJnVRWn2aK56O19TscZau`O_f#2X#>>Q?Eyq$pN8}!s*)dka(;I zaDpk!EK1nb+I?1g)6zqTn(moc`?4J7Dc&Ljy#d=q`iOKC8d6(Tri>{!N*Z?%+|}E9 zQ&UDSpuh95kis?WY{j@VyCJz6f@nvLX~u>{Z-V`0xG?m^^gmjHn_4(aiSWCyd`YmL3s6ef{R2uSAN(AgE(i8^e`N?7Y#k0g^18Dc zkP^5x6f_n{ApuJJ^}ygjX;MqWW9TbRrL6_~VRpM&vq#HZf*2e4L!)9lfi873Lp4Dq zy_UyCf*KM%ze?fK=G^lbT^TBv6x8I+hd=Rm5MP;yPS#puA=f%lFz!`6%!jkurKI`L z*Lo|BU~-k0u}&ENYe%u47@~gy`@zgLdAOjHdJz8W(a}s=gVZk!7-3Wa@>#5C5MS{g z;yS5=aWG%-I;n+JfqwFmOF=-e_)5tiXTHyJ$-gbWm3s3DQqqW@XoAF9?C`bX!1)QW#MHJ$k2U* zQMU)EutY>zbO3B4b;XCy(6K>mBh9HM^HZsDR?~0^ttCHtmb2K?3_`wlF=W9Z%O^(> zMGUdW8=ID5f?@PQ0L2`(J{A?X(W*nhqS}O2y2`rtM%^+NaZM&5OU=U zl=jaD(>@4DO;CMf2rMwhx)gMnw!XR6CN;)7s*>|s2ED*IY9?Gsm_HnVk@P!U0(QEU zKO15sI7nsY&V^!oP?a@seUom+=qB!-ih1#TC{xyAvNmn>k`7TdEH$GM$#l+UEf`ell&LFN!rEoKWd zWj=f$TvgYrc11BS3ZhVS%3ktip6Uur51!GN?>9PQI?rJz)DVT*UAwU)Ho`_ElrCy= z*YdC0ID~z!+vYogaa&$h88*%Na19xBQHadx1VIspr%*%RvptxWZ|n0M1*WalCQh6! z^_AWZ^IT<&%F@2JD`xghkr?tn*_*W8j$Os!Rqn7&Y@p(b)WJ^6RgRSt`AuKqx7F zzJWN1&o>=QopU$+d3^)ofY`Eo2>Vs`r?X;LMF$dridt>wZ5OV))muAKi}+P-O3b z*sWvpwlCnlXZ?4{r~lF=0^`fHS8(*MXDufd-0O7v`|hgY6`X6!HHc5V@uSw`=<=9= zu4r8E8;=3uyAa);u?!m;oYAp^tihARTR3)cocrQQ>r*%X;>iTzD}mFO^@70yyG0IO zW(fOLg_+XdZdg(g{>M9*CO)2l-UBsW%y`MdJ?pR^#rN&|(S92E2-270{&NHPs5&np z9&9Q@>P1nUrUg1Yx1VyaarU;Rt7zq{w(`(()oCV%sc+!#mc;9e5z%G%A5s?W;S}w{ zGh*)iQMy6wZ1wO1-OJU~EG}mcr&K;_9~r9_s&?~t^xp+s%m9vBLwj!(%^{O|xLK?M)D=c|7A_}!}ooBx35 zMMj7KBFi6IWh$jBn@ePCb-d!14iSfRWj?KFt3l2=$b25|p~U^q-4Gns3c1vCZhGKX zOLrYT@f)%6n@``>Kl>qu!@1(ZHFsn_!TvagHNE_ft=VdOD~;o|W{Q_P%5%KTkUbzX zU3_9Ri|RZYQ48KA#F~NZA8B!wIxayewVg{EIYDvYP`kjL#f)1yPhoulpk97KDdcA= zGwDFinlwCuCZ7wFEgz-@{t_~>yBa^!%v3Sx?rY+XzKZ?cuD4Q)v?J3lV3cZMGgfBX z>0-&))vK@9v>d*L(Q7d84A0w?v0FJS9Bit4Rq7(Hl~D15;X$Z+%g8{`YkD4qehK=x zvy<7=L8K)7=Vk~h3Qn*Z1gP3jV;eO`fte$4H);3yWL5H=FM=@^knvwfEt zQ5Fg9A~%uQFNSR(_l82tf!*F zEuh&~E`x6STsd*0=oeK%BWFy?{*PS{;?iiij4g1-V#1f7tMAHC%IsjL317lbN%?Hc znypcu@6QZmM;{+)g7Msxg1S@d49Pj!(9;lp2S&KI<5D)BBmFdc?P9{rk59VO##h_zPMOkV{8)2o{p|ARw26{ z!{e$A4H`N`xarnoI2bWVC^jg>JYvckl874**0mqkm_dx%FZ}Edxtmw^2p7JXoKuir~CVAsF^zoF>wegu+@hZ#W_(bsGKBAzan34dN>K;lP4 zG2i?Bu0`-?FUMck*2VhGlA9Dlvm*5xI}bf>8ZUft$_N0;rKI*d1!qVKeHjOB7@iTry>vKmcDAVj za3I=>=yOVe9s&vl27ngK1A+ipMr#}xBM0Rgv_1=>0dQ_e5im)P3g8tM5x}2osz9^@ z4+atixY(uJA?M`Wv<_L~YwRL6@Q*)Rw;#{I6B1tact1S7G0x>S1-Q(eHcAAFO95wWt(8*a(~hzrsY zfP;Iv1HKYBK<@DAPP#H>M}YFAbAY)JJnH$od_brMokOFXwc~gJ4r68V(nexxKVM;y za0e;`m8b=k0IVnB&w36#phu7s0P7Ko3%~=chiKSR3p39|Su`NdX?rp-ZWn|R{4RqM z>KI^3@S;P8xYMNB6uWD6Y1X3c%-^1Vysw+;7`E)v1li%~bqzRUkq$Vss>d0)KNqPz zTI-txu?DPYf#7iyZy`tTN}N-76u7+<9JqZPA>e|%A81pyI2&s75EGIr6EKjM!6<_k zk_J;k$tzMr$urfi`l;k>b%kOUmP7*PG`*j-p*Hk{&n>HtLrE zzhl1rVtAUicrci>ZR&Y8Zt&mR8Ad(Pk2^y|W+P&|qIhE9MS$S*JnD)f+3N60ld8ea zYW)@UMEIabMAsBqw-Ddc=?Cjn+DJ6`-4xOmSDgkfu@vwfBytWg!=a=qlniQqT7s^3m~aZAl3h zJemK_AR@l8fg1Q=osDs=@xvz@Sge?cCiMiQKm{ok1@Q4Q42T~N{C1?j-#V%bxRT;}!T4mTD*oLHf34Qxk?tL_s`oOrR|_20S=I7OC-P(@KuWlA&K{5$yPgvq7{_A( z)t-U^(N4opfKVcV(`*CSXf!=f&by0%b;C^3yuklfgEKk5d|MhH-R)1d*U|`@y2Rb_lZxu7$81oJBvCJ zdHQWmt^-0@z7AhNAlqGEE5}P9Xsu5i`hY?|UhK%_tFa`Z<7o)l7yQOKhX<>jF}sT? z6Xw+?i}|97t)U(omLG0l&c(f$N3VjqsYlEVPJzzNYkwc%oCE$Hdo+(}fZ)}mIF@m! zT1}oZ$sLt=QrwPRyBRSRSDGu^)MfZKXI%CglckU08uveAV<`YQJJ!^rAy<;Dco-`E zL}2kj)MZC;S>>@%mz-IL5QHt6246lPV6yyO&n8JyZ!@b#C)wGbexhOB>fwB=UcQFRFhip(xkQ= z>{?U##rfj>jLUGpy0O7=g*|V%(->qZR5)Da^Tc=@cA-0JPEvVXMYs$j6~DJyjRx+% zpmzXUx1X&n3uX;4}J&q$-qNciOMNFy5K zv9)$GVw_!eE0SfuDV#fpj78}$MUS2sW)i8QZC?RU^r+yXVpCv}e-yo_BYf%qBhsM9 z|Gz;RLNxzpq|sM)Tdl0}OWP<#c||YP1@*M0^?8z-vtt`MR|5Z40mQTUkCKNjdKT6a zdtyXvCO;>#=-0J}H17XyA&I8=Z1L&-1k?tB&a;=gB|B5mX=7CRMt%hKOyD`}AS~b# z%#0Zi(b3RK6;Mx}4J&7drssl&_5CvT8}@@#AoG|BbF>%}L=0MtOZjRyR=J7II-BC~ zl0-Xuy#~j~+s<*2xMbg7w1as?k1RiR3XKx9f`ZSjT=T)VGTlsDwT#|R)w#pPDU2#bU&HG~&QE*Hrx32#!uw|4La(hc##`Y;cKV^Qk@O|QH@#h2 zLVDh34S(F>x2~Vf4etfwL8)1scw0ga|CxC%kB0Kcvbincm``_z&R;U0WaUO-q!6vm z6z}QD0!>Z`-^DIFP7&Hmw3QTbs!PFBa>znWX6*S&JRWXpGyrZfnwh}G>2c4;(a(*W zEIHcglWz5ZAkbwJ9JXI&#h2RQ(lA+ncvj>))NS4LSP%s5o*&=D+u4L2q`_Yr=XSA%C zCaPYSSpR3<1H0RmCZ|gUYArq&Hu1uTaQuPjucign5%|!-raS2Vw6Hik_tD)@GCrwtun`o;APzz?$>#7;|Yo+VC zQrplL%tF{k`fKHE*zlKN7-FF!KREdyPEGSY|GbH+pP(Sb5KzpxQa58mP zss%6CYi#i~zI4p;AW6`7lYH-TP*S7j$k=KBeR!x`gzY;c;LLVyx{8UkZd7b0Dd+axq)3PF*_>1IA}O<|6LY&Bgaj&NrS5#?o5MEHq&$1T+Y#P$|>;-zDH0I= zxAY>Cu!>=yP^oDT!T9Zz5eTS#`Tnwj;V+?)sE2*%u>Y@8qbyuHUS9yi#h0T zfv77Hs8>F$ac~j}L^7vvkTOS@YU|_#xStV{#WhAwK|EB)a}Dgy=x&&A%nd%Q#|LA# zlix~sZ&4kdP(6>=IOM-)1@Z*;ST^@y1@x3f_At4qZc{p3*Y0@sYMdw5Vg=Q%VD&;x z@Y=0WFxx#E-Jvq~xI?%7JdLsJG%e~~3oRTuJJGwEljU8F9Dj=3Br{Cpzrp;q!|2A- zg5+iZ9l^G20z|VaIPCi1vyb3;30J_PLAbOpK32)%u; zQ!mv!?^Vq*sC?sDJ}D~}7#s7X8_7)LPvZ;Qj7}{sZQd5{Uw;vNdMvb*|bwD z-tTt7fdNTsp!>;P@pDgb7~MVFY-MSqG%+wr~SKD52x!Q$5z zjhcM9@gB&k*-6tb$e6r3BR9qUEeR$ z`r~N_p8Q`qh7`lDnGH1WkVKf?Q1jk#@_gT#PT{Y_cbK0(YluI#BnZ;+6ZcQuGm`!_ zIvK+^OcseY+7fr%K7L~NC%Ys3kUFN%r;Lr_wiYvsR`9;ae$uGyF+uSwZ}IK{%O4)g zA1;!$RJSz_JvKR@B2(;sWFDbG+-_`Lz#&O2p9nDLe~0NCDTqfTmWGlRk#asG9-TCf zt{%jzB+s^I*=c=9Zxx}kK2!WGb2LtE?3VwDVsjOWgF+ocVYCmKf|o^Py#i;N6N01_ zLD1XFHt*r!h;tJS>*XZe_vn`?$v0B~X5%4N%p6~Tr;8**6T~CR4ZliS$QrFGe77>$ zDe_^ZO1IA3XPnm`#Kt_7h|g+qry!aoLatsTX5r(45Xm|efw%csz*N-h;E|d+O1hGV zT_}}w!1Ar&Ivs7EHT*nv&;5F9FSnRceyXce4k^)CKW^Jz@ZPO$^id%_0zXyLUm%CV z3LAA>?27THTD0;}kDY&2)90X=yoC zyx_h3#*6n08Aa5C*Wdl zzQ8ZW`<`cMa}KfXGT< zNFqld48+`Ckk$d)c$};E-*yedVL!HD+@N=qsk&8#4&RW5m-3F!0lggMjPoI2NQUIL zDj$$3IqWC(`FC(K(|LDQrfM zg0TUHHaY9aikz#6an-*CHiPXhU4bGAng-C>D!e;ai;w3Yd6G|*6- zR$s(05qP|XUSW8=K?!0G?c|l?heUBbn0+^-=Z&#(eFtDY`TeZoP%~aA>!rIh4lYyb zBM77>Tg?a}M#r{L625pWvEq7bA!P!zL;eb?%hB9You`MS`=PnV^aEom|KMGw1Z@S} zUEV6(KFPFYeabgMHNK_JJ?1$~=}8l4;ZapD?YW-*V z&xGH%IpR`JKk;;e#{|pXg2#H$Y5u&(5-fWjCuGTAL#%kus`J_SX5FKDu%hiaRo_Il zfUakk%9c@vyZ}s~5CeM(L<|XW3Y3q4^v?&4*S%eza$?0AiVOEoRbd=vVh6F44GO|C zfkPK^#kL`o4*Bj&{FF_*8>ds%s21q^^7Zwwq%KfcIH6I35KhkT@GB8dW+u_k2z{!; z++N>J(>HHXS(23Q%I4`uVl&Wz{9Hm4HwPUASC7LW-?+7XA z2o}4AhvvWWe?#gu3|kYYy5R%Ag!i@fJ?q+Et$4G-2Gl*pAPIi5B3^a$v38p^X^NPj zM(v;AJR^qRi@bB9p=hd75mmFFA;OKA%QwqP;#@D7OmqIwzgZbdk?L#|x7^A5Yc$x1 zJOJ%ed~79f<2(K4;jDtR1vlQ`JxtTIDRyIjK(-x`EkZZT+io@3pL;{Sthxn!>sO*+ zHK{+)@zf2oJy=-^PbPS~9N>5UrXmpMa)t`5Pny1t+Oam_k@(C|r%bb8Dk{&hVCwL6 z=$u1)Vsoe~o9&rj+J;7Hs~g(>n6=#1)66bCsV3)BkY9Jb7+4D{;YSOg7>B3MVr z%_n&ouTm-~;ZLuyA7}glGzbE7%}C4)WL!t=Bp{9)&>opPX9^qv8dNywTc2lr^HYGr zFRtV~LNO4vhlvCDPT-#JB;4HDNs7<=_Y;PA0$!-{jaed-ZBCp1hihS4ITddf;h_ve z8Y8Z2Kk2O}B2px|$wS#PeInUndrTjZOkv-fgb#S$6GG54|0y~&6w+XKmzGPLBY*x5 zE73JIN+mq@W?D;Yc3KZ91vc@q%p4t#ro2iEjSPjla0MSuynuQl8Y~5191GXxyeoOq zt{F((_Tyj;j%JQEGI*4L5H`y~-Ajq$m9@vnnZ9gcOr=uHQj<^!c(c-B#l(5V!sudd?feozdG-!H!V?N2W0r9TD*A+&i zsN~D2Y2Mxx{K7LwEN|H&Dm+I$UQLj8PnA(e(x9-|}S`tGax-a7mor4yV*pPrw z+P7dOI;sW~Kq$=2pK0J00<0>1_CVV4h{rl z5)mqe|I8XWuulEqL{(UsE>vvO)-%|25iKcQ9uTlfRr%6mA3-Z7ei=sTGHX)~vR$Aa z>HMSOw!Dt0`fvHF@vVv>3=ginps#?l(#(P?n_}{uE0XH8P1LO;{=};odTR1Y&_Pjl zl@GJSDhSi~(pq*>!-;WLOR5Kl4hO3_3$L4&a6wcb{ZU*Nb7QB7m|bm(*R6@mxS(98hbV`aexA zRMGcV*0wL})}xl8^(mI%K4_n)Cn%u5O0SG^Ws)h78wqkXKd9KA_7rULEV9+>Q*>-1 zBhaw}Mubw|>aAO{&~A*YHIfO?55i;G`KQbF;RFulNsbCC_E)u&E-N-N4cmoeitD}( z;dM}UY?|4{H;VR)kVlO2Dc4L}%lykp0>174g&)S-|5;cWpYYDl;J#bs&BNUbQQr@A zZ40;u#LsNNuUI9cD`%g&@GDp~WVM0b8Ljx35!)uMT`gBukOpw=ja03Ql`!yW57?(& z;2puD|4`@HVLQPPge-Wzww#+#lV&-ST6C(6Xw!;9#n6oeco~L`eUm1&|AbZ|P)f9L zn`S-M)$jrAg=(AT*C&&`fS{GH6m8H5q?rcTv66Zpf3gc)`G7R#6Csi`&{oDmeXUdQ z9Y=9EG7r#c1_U_hAOy+eKn)TcRZ^H??@}-@c@4oH@MJ7xN+>7<5<-BZPzkJIe4wKc z73ev0l0w!1fhzK#I6Pvb(xK28Bd!$Dfbdq@BsiDpKQdG}Bg8*cqXS7kK>(NyJ1-GH z=a!rUq)Wu!gsMsc?~97LB?f(D3EXy9;};;3;ftrLm@bNwAqTsfr)%1CnPv7Oy7h65_Dh`RS;>w%2<3A0ge*y zE7c0ZibC+&n41yrsxahcs~eDfU%(%~U_h3pxq5sbq-`1Cp4c9H*Ab5v=ZT@pu;q)f zo&uh-?aP%4GQDNb7-#t&d~P#-b9>@h4E;wK$}#L0-Yq|kc-Xm5oQo{GPi$Qc0NA(q z{;~R7z5;CjtQ?U+YnJs-dEZD9#dw{WzIVOlxeRXOquaQ*TXa-fEHnH8_CXy4p`IXw ziUOJjr0XKGYh2|Ic|WXew0>(@GO6`q)?w9QQK9IWML2P-Kqdnx z2a=3L1;ecXeijr3tTSdkCism;4br#yI5g-ZOD8jed7P z9X10pUM;6eR4w#LVb1H->FetF#CaFZZLTfi?!7P$)6rmgR~Rr3(L??H>m>Za{p;G8 zTNs`Jbr;QIsOq#Gp&Q^4U!;1>N!0VRO$Y4?vK$!k-#_P#m95k# z+Mh9}sjGO%`Hx2iX~cKD(NmG$<@CK&n<^uskb)4bgrRsw5TJNcphRc#0rW^162PHg030f;KOHC(7{;O< zhzd?gGvXW+EzHcVx4%ImAoTWig&RYmE(#P+0Tu)=4Id~Rkn;gVnmCaW82{vae4aId zqHk{$9@t5g>41r$p`?YRL@1z%BO6z0L{WtD;h3Q#q_4Wq2$MYq2^x`nMtXRm1jM== zh@N(Yu7HCOU7-X3cVz@{S9osdl~9Gcp%NI049dxfwi^M#O~&&OkQt3Fs5}QhLY=_w zq}Wd2_Y!d6_b~*a9`gPGKL9O_@cVOXPWbO$pSuZ`Rv)?7l$d^sGy)4Z0vEumNJnrK z4V~iI1@My*A1kdu1tC29o||^zZ#trej)-a_ksvmJk|2`zBY9G#+8ZAT`#A~qmrE=r zUSSg%3H4_(06~MIV3529gc!+s1oRKZ0>S)xtgpOx7aDLD78+;(opFq0%G(71%YxlM z&^4U#>Y4oPS~BUE2A`{h+#9YF|A$(a@Qvh4@1^j-h235d%+S1(y};L(0>bzu>3190DROdA^?a=o+ z8*fJ`*_)RUiurNAk~^!te;FmA9bW)=CGW0i1HrkZ2apyinf|{>i-7(+ErPDoz>T36 z3Ci&Iu!vBjRES=$RG(CO*6--VhWHNr*9}uTm&8cVpvj;~2KIH#YS1#Y3IF6QS?u}$<}5D1es}UXPc(5ZIy1;zb#E2wovr!e+c<$<6w9G z9kZM*0Wb?%(##0hD@0c$3Cd*%pA0?pPvl8|2`7;S;pOI&r<6R-kk{9p4ga+7WqUt# z@xULaAfJrq_<1|N>FOt8rqlF^y7ILWrJFc!bx3ivMHgiG96CC&<+g}{9jqSxSI&|_ z3E(Wnu6s}NmCG0KJS~i!X}H}%>c{$B>Id=bh%0ObHZC2T7te;hAKSm(=05>Bl2^=k zi>|(*tIlOyxUhGN%^w$a?f}ff`Lr>+>y6ei-LBZJ{;-SV3?{Sm{BjIwjEt#Tkl!m? z*puqF+vC*tzxPr)l{K63_OaWx4D5XUe@K>CHl`I)I`Tsdtk`_Pz?_|lMAmZRw zeHBjqN??sX$!@#SD-N$m6)w+W`ai1%Y?o`(oe$&Fq~<8-1!e|?Clg8zZm@Z4M$P~9NAxy)X`Y~IJ& zyUKQmEH|^ymI@mE8X-f2{Q6KyiZ)HD1=sPv$(F=9#}^rN@?XOTPxjB=q86oSX9sl? zoxY?+O--)|y53d`W*+Az&Z#_oHv}Y0bS`fHZKi}CrTm+jvZ1`yeMo{VpMILwWi;!W zAjplj$*49NWx4-C^gc`uj`|OtQTLaV3$IywI{@<)_J83Zzpf4!WqL*Bjcer<5IOSI zyEdCOo<+eFkSQq~#l6mK`$I+qv;J~^9TL5aH}k=9z#u+>6BecGn;j64^we)0jTi7~VNd6=`HYHwfZIR?!MEoJVv0n2KwY7)l^6SS;U`FJFVlcU@CC5+!Mu zEmN+Pzo?vsLpZ8y}#F98m>xuBsQ zaLH)!%~crK`H-^|DE=nBa=9KYshzDjVd!>p9}z<&t(yER44w|IcqqMwQ)Kk%6Y2JV zD$7lTwNBPfO6QHw*TFF(b-k$ra!OfrN~lq5g&Bn?g$7s-LOOlDBzGNo8ZPKdzA7GJ z`jq!bel__Dm7Mc6(k6}iK)oDk)FAbv-x^CO`FO17fWw^U>uS>pQyjNu)bT%q8%A_C zWH-0$Q}>uz37%HuVy2O@9?L7IZS$6F7qx$!E%y`{1lOa`jLiipDO2z^>nbm;piq@j z1NN<14b{sj=+B;pu67qBLaR7Hj_v(8Le0^VRlsN`*j2y_S)(N?V0GtO8nD0(G<)|9 zE}j3X+3^2-f85p)d|zm96aIV}PPNkaOf@VDrY5gx8gmuZ?F(LgUIDA-eER&^LbsV! zu-vT9f5!+{y_X^b54nUg6A;DDdFnuKAVpeXe?xXiqiPo{2}bj@EcMMop^N3}l%BM) zYNNH?h-k9A7R?SG=kpYY{2n9GLqyhdZ_1#jx~Lr8E}6g6_S%o^d! z-50IkuSYLY)51)b*L=LGuls`whL)=v6`=ECTWAk?s-K5PT)jS97NHCn zQffV8=$Tk+;aDVg){vn;}lj)6G;<2_ON z@7@yuJ%9Fa>sO3T-Uzsv-^i}q91-`tA#6ZHpQ@cPr0HBTE@9PASz$WKK)grO{7^{u>%!FjKaz4P|* zuZVXOE>|caT^%^ZEy^*qHP?w*@++_mK$?xpq~P7#pl0d z2_b)@g-7dyr`dL}ZQ@S3#FQ3&{-_yy3?y&u4g78O?5CX1l5$B(wathTeDm%jOtn3o z9(MB{WD?7*xx7JLz$r%lf-hWKSL~D>dQio8?W<^PYft}4YvFy!n19&$wf-=Z2YeoQ z-pWz$lSBR&d^Ig7H~MzG-Pr_f#zAuvn+i606qOQLJN8MJO{r2RQfcS zDTuuP9L{kdIAbU|;F{uSfr5|mJh*WLu0_DA7mLnXfTH`CuH_H1mW5n*p66ZB+P9yJKL71`+NvzlFw< zik1;DZMdmUPt^cO+wRV``@=udcH)n;?Ztzc07zSqTY$6$xz)zZzywI!?#@5bmN6VJ zNMe{=|6?1n_6dm9P+90VIqFqAr%a|6N{FRFleO0qc2zZh~HCbBj@o?5#)H)G{ zq*UMKU8(F1Ikq@8F1^YmULfLa#0eL1%;TkU5J1fZ{ZVs1f7IL+K+OdVmP@1(Uy}W3^jlnIsq)Sdw(pndjOxPc5*EjQEyaNXR^{W6R9?VZoC@vx>E}>pw8ff5ffJ6pGsPiNn#D~EV-pEIA3O}#l zZQoUg=?iZp0wXkq1S6CQA%0S#*&N&ne%k{IQ;n?%-em93wXwU^t4p29BkdH3N{HLE0^oC!Xo#lf5rVa;s*~G<0mITttE=nKG9V z2B<<%$AClRzQz1U6~csGdKdl>d9qZkQfowW~*S{{(2yizuQ{&G&cM-ctSgG+4Lik;0$cC6jGwf_1Yd+^H^PbLp~rxe)av2RxHbUjU9 zpQ}eNTLlwx(KtaWp{To2Z>XBoP$jcIu!kJ}l`BLNhywrR zhwvSp2caCRt4LABCyh0K2NN^dNu%Rj52Z==%F$T>@7w(g=p3&Gq%WTwKo5&3`gmIW z(ZgVJz7WGW|5XrU;a&5HXz@7lnA=f~A~)n^rFD^|{t4gqs~BG)8Yf;|`iOBQElX3(Dga(j$B)(CJvc zpp;Kf>sV0|%U0RGhzOFDjr}lqrOvwAw zm|$SeJUioheHd1(xC-OI80T15gf@$K*6-vRt&nh&y8JSxI^&eGO!B+_kGvIzzk36t z7B?@4FWI@@@)(a5w@b*n?;U^kehkK`iho(# zD=ZiyS;#6hO-t@X_FokKaVH7Y)a1DHHUN9~E-SlnU)sikbCR~Ns@UfBd98*Wt9w&g z+adP${po0IQI{^h>$8>XOGJrk4%v%UeHqL1s{$LKL$g%X%o_Qnutb*`Zc3GAE3}+s zQx)RA)-Jl;Er6K|&lI~}SHFl3;t|V8QJnjW?yCS7zoObxRI?GkgXR?Om&@+lQr3iD z(W;5^blMbO!@M`=saUyQxZ^sC1{g@jIW18)JU`iF2EN}mYR!tG$)ouH5Wl$EZbyI5 zW>o%hzs9t$_J%@XQ74F5Y7|6;DPI)mrs}|u+?qMME>~<3byk$b6ke)aQT)WMP=@xL zGH#XJ=c-Db(6_!D8Q1k;l`%cyz?a%Y#ejTB(VZK8ATk8-;E}8Op}jGy;QJq_ZqD$R zbZ9);udx=-CV;=-(6_Vzym|X$x3}7^38luM9dxzV$oSvBft?a3(auS`WV&#g2`tmc z8_SXd+scsFQR7#C^;BShV1(Ge@ZjUaHGrcqX%bW)#>9MQ{etT9=St?&iOF-S5rRlkk8JPt3*EfIo`cEF_!Qv_-jfPg=A^$ zv2$H$`>wF5Yq)PF`Cs`A(jC|$tvV2?A4k~aE=sd5iv5fgVdJcQ<4#?42#kOTj4BEG zI)8@zFivA(d^F%T(q3qQY~&w3INRN${ocS!gzq1m`z)+u;&~wnTfU!Uid&8i zcy?}Fk5`cbf*l+HL6T-0fyvNluO*p-;+4A^1rU^V~3>Vz7^3o?)r;ZP2PV93}1 znvu(0Dr!=~%-E}A2Ug+PJ(@u-V&F`oMV@^OaZSJ{GXbd7pl%xrG$5=h;bnyAt*H*x+41nPI zjXrf_hT5lXGbtjhYT{FmlS!xfLWPs*tLUy^FMW;J9`cU%PwQs$Tr@u>5J4%Bn>2rO zdo6Dnr|)oYSIf;Ce7U`5)6z?Lq9}+Vs=eYS@(cQfk1)4*`^ScBPQ}#M6D>cU@e~cOED}D$Zyz55s{xS>I!2MFe2m|V^M6!* zO*17B5XunOXHKU|loH7GS21Q)FiZG+dEmW6&1cmS#dnz$oVoUYT80%slf}>ov+!_q zb8`}2kd2{yA3gBNlV5<{$cCF{n4W?N7WlhKObNvLF& zMWXYFRXLe7l+kI~-ueeeXy9I->44G|Jyxke6TpFHwMpn`8_z02Ui>-z1FAv#sF-@| z48Ru3pq(Td;ra~>=Ee(ZLWR~0JV>~=m&V7A z1jZ0jU90Cj9z3P&s8k;t@4-AFoJr(CpLMJ6j`_J&Mtme0?6-vb#9R7o(ql!XI;@Ul zdHA-~q*Jf6t6PZeE2d?GesRqG%P}8XI<5zNY$HU=wRwi;l&U3&BdmmR$@^ENVV*kv%(EFa` zC7|YLlcFgO)O?`hDb_^)yB<;f$@Cak?Lj5i?LM|^X(eGPq~tkb>G?1qvGG|A%eU2R z+Xt&Fr0p*4Xs$BVciD>uo_uriyBwqhAscw*cGjXs&56FK6l41Wg)PBYlMUhMYvxRe z9KRR;mB<&_rrgu8iZAAvRp&Jcxb7CX-ED#AToCR=UK;L*DZzeu=Qv|EXWDM>2h;QP zKHh>bl39;0cTbNukC^SQyOW}&7j@sPxVI(Fm-4@L)f}D+uH$Icm@yV|uFBKD?w|I& zGc!Y_YIrjehTd<#e;9gfjW77;!gp9GeRjo?)_!FB@5(i`L|?9V{#<*Es9eJ!$Cymh zSyz}z{%sCh!TU6XWuU*aRadDGH*7j{`4Hr9)phYyu>B=mbk2J=_s(&6Fin7%AQr6- zoq9*Uk-lcQ5m9Cou`z|uuclRs_nU8{=qDM0yzuX2$}ef@TVHflyO)BTKd_Vp5DxeA zB}$`*_G4A{ldccqMkC)HRm~1OS<2^b&M*4$ip@Ap(;NhFH3}>^qvNb6SYN#@DkhV0 zr7TXT(!CbQyCQ{@wt~ZU>nGBB$6tqP8=X2>W+U^R!jn2)eV`1}w;hk{82N=BMpi+y z**WX?kJV=p*39_HM0vqE$7+A}(@6E1Omp~TwQPHlT`KRTY~aY8r~|Gx&vl*ZXR(E~ zi$1WO>*>sFIWSSs)AP3bCD_HHS~DX)b+P+6hK_Ua_RwbNe;009x!fJFp_bikcpdfT8=AakfF~r$p498krFYnO!4;y{fpjrO*OE(C~>`l zXPW2Y1J5T`Bhl1Ls2mj5I!XWLlj&}w5q9WQEab0H`hEI$ax^iVN9SzuH|RN`qbjxa zNf%sUf{WEu9^omlnViu?(ZQd>EqXnj!<-fw4@{l-z0zlfORxbOy{e z#bUs!6DgEcR&M>PHQ?4OGiaIwq?QW*W6B{x9x4f2pX?h#0j9rKB7aDVLuETnCcI>J zW6w1`Z7<8`0i#^ItDtNBA^2CPXa>ZRdNf!F@EV-?yJ|Bun4c4^kXWpo+~!B`&3hT1 z5GLkyFec{J!*c1bw}~b;L#ahJI2@?aKR?sU1;u+hg}GZeRP443c)NREVjcmuF@SBr z70$9Jw0)eOvYy#L2L21sp~4klKYZ}A6YEZ zWXs{o*b;|HY3ul@*ez-_?{a1g&SJXDZtl3bX_s0cq)u!ox3h4*m_8MHaRbyZU~C3% zb}a0_&|uphlgadi_#-9bEU{qPHLbi1vHDV-{i2?+%sfXG3FDmyewby>U@i`mj`ll> zNhP{(Ta`%vFWTNQIMT3P*N#20?TIz9ZQJHV6Wg|J+qONiZBA@w_q^Ee+Ur}pYXA9u zrSf!DSErv!_i^I7?m@|I2hywJmu-IuVU}T$&040IV%JR$?)zuF|XrGL2|Z0N-=3Y9lePcbuT6N+EkO9q#KQHxif7P$%*T0#&x zh}&Q=ra&PZ6Ek7S9>w0-C9jH=cSoTLg>8I56~}M9$=RhGbZmZu+po3_Aa3DqI{Mo1p zJCCRW{UM1e^Y4S+CzBOA4>JE5Gp8I>oZ<^q?7|%wN>_C{bmaHZ;8yYQX)MY1z4>6| zyg^u9@Kd~79F0d#U;KK$3sfFdQ}0S30y<#c!a*Qj(m}va>`N?9{k~q#f>8M*)S}}( zU*b8jO$9In6Od#e8atpLX1&O1ptZu^YK6b*GHJR5oM?!=qJYSOqzQpgq`u*Rk(;)Q z0oF-Q&gSqom)gYc+q~W0qN)gBrLR~Kt<8i_2V;Y?%f<<$#})yVlK6274hf~CI0g$& zf`frTkun!_Wm@wA`H&9)d)R%7;0P+<83IKvg!DfYmQ`HNn;tz{b_xm{-K0j5v z_cJ{fl2`x@NOBmtLTC&pPafzyu)zB4jtA|KMDmy9Ka?Qlko=5e!GHmddobJvz{`!5o{W>e$zP$87j1vT;K%Z10x|&_ zPZUZMMHU6}mDue41i{o#1HwdM-FTDW>P`I(%uo!$a5PJUVm5##AL-a2$j|6&2s8u~ zUY3ccYEV>6!C{rMZc^#*gq^?x;cKSS>i`OdF*^<9y%+3}?e+k6(vWfm>N|buH(l^! z7UBcVH^xzDr5%`{m&yRPm&#WKf5<&oR%hKMU&cvojIe@?ri{|;X*d2r=LfiuVBa2n zkhVGxFi_o>CAgd~qQ|Q!knjSoFef9}3qQC&Gr7*Tnjqw! zn2HI_LRRVmJ&QAM+?ju%GKf|(BKTIa#ae+@leFpy!udEUxJ|97oetKSLSTvmME&7E zK3-3td+uO=uz_gdiNe3Ye$@fq;vJF6U4fhSVXEgH+>08i4Zd)$WOPS~$}jPX4jX zy$^mKBPyigCGxKa{P~6y-=g!h9o0hf#6@}4eS#j;=-_V>lm{S&GC81;&Ddc zAq_+*MZRrS2X@Qk!E@_9yWs4FtlT0e1@fu{_F}KDLY~HHBb`x;h?X{7yBsio=MQU{ zu$p=m3J^-Y`5J5lHhpAhK&S$!7_ZDpUIXWzmMbvR=ZEaw5_SrxN+1dNYZDprxgraW_p-zFA}d24?D$71+2yygQW<*R46MtaE)-) z)XZf9c;PU2|MJ3d+fpaZ2GwSDOoorE-7Z-cB>yo(f3%C8cx@m_1Y#J^x^7=Gs&dHu z4j7>i7-gjJ=7L6>U_=FM#6wOE`&;qiB*iZchn*B*AGY1IL~DI1jIO*VQuE8BxH$|6 zTLWwZu5$b5HsA}OzYaL$iHnty+TB{Qt-KE4@Rom;yDEWdEX?aprjA1~@dYA8_?}D9 zGJm(#r($|rCIY94zYUt);RkJMO6cV|Cp$hkE*i7(>;>G^UHx9>B+$(yz5#Nxy9nGn zR_IKj>P~ey#DX7Es;2vb&*5117o6^xtANu?a-oe(=y}2=+9tVrrd1B^pRC{F?#W`R z^XD)!HXw4>2nkDiP;tQFO6pq^(JlZR@BD(fZi`T(1E!V2jF9 zC#IC@mT8$bzK}^jZ8&Z2$S}KoYcCKaB_$#Uh5);8>O8aj`qmdg0Zl5PV>Je0w&q#} zikK1r>dDLAOIh7jJ5$o8vdkb*1Ci%K>w({hm08?_Ripv2(uSlwtvzh`|eIU0sd3uG9ScZT^_l z#_70TM}14lT+7IkXO$G%Ap+G2pd0zSCPte%Fm9yg_HcGY8UHsTpHlI`6~`J7`Ll;% zLb!ii?o^b~+p`tah3@o4)_T``!rgAK$mQw_mSg2zo`*f)W ztFH*=dA+HjB98n02sQ?s>Ri zZeA3;t3vara1K$Z4b$rSO)&#vErr1s^xPm?VI|sHg=*mw>G(EHKzo#O9cU~P5g$Zc z`xpPJKhosP{`(t|34FVtpV(Ci#3LeZ;AoG?dHgG|F%+hc7v6es^I|=mL=^hrm;Dy6 zZBEP7;an5?r**W`lY#HNBgW3sR!`qx7f?zf+|3Cd+@sf^L8pJu3MApOrC*~Z-`v#U z{uSw%$hfaTe@O2$r zV+nY>c2NuAKX!vq#!qL^L6{6yptyqp??5lWw(0NJh@QYbVu=M7vv)5XpWos98y&~q z)t1sM#Qet!le}GfpqviE;)|mbCY}5J4NaBKYz-=>^fVF;w26FI z-;ab8igVE_P|5xI9KkgK@zAp{ReUs}Dk+$6d`B8+7Gt;pUKCaOXxPXr&dEk3W|Vl6 zD)d;SP$eqWwl~Y?0tcTK zHCS^f8?@AdB^?>^FvHBrB=s(YIsr7vh;DjvO!bD&n5_y)NE}_poJb=QVB()go?9D; z)hV{;FX^YPZIDbCxX6?%?OG>Jm@b|kp1qeQ87i*V$5K=%1jU%qWh|a3*NI9|dPj9M za~rL8;?Be(*sXpVKHecDUR%v>G;}9RGASp;JgX4lFj{s6^x0nquCnHAuyChLycc=1 z_SB#jxS3K*M;Q!7VnD=hx`shIpW+_6H5~NTd4XBIokOjd9p7kf@>1(mAQZU!U`fj} zev!UKJAs|w?axIZeHUj-6e(Ben*7rIx~6cwNNRK=Nef6c%$K1-q{4~3MN`m5qe3Kr{dn1uLA}lF zxSUnW_}<~@Bbk%Jg3g$cYchzGm_8Ji^2096IP{ztRVLV+NjQP1AwC^0?I?FF_Es!` zXb#~glQB8T--ixNVu>;ZkY7_RSr1I=hzRDr=`}=u3K}N_hGJezHHoWk#vCpK`=-6Z z^syL}p6`eG?B^_Cb7O2t6Kye54z3YzOrh=7OAsxg=Q&j)s>#48pc^YNf z({#1L^di3dVJ2;3tN1i1pD<|J$*WaLd)I=vFl*kGZ>X!x zhVJD@7ahskf#;4NQ1<>>6JxQQ>}MTN3|$PumBGmA`F#CTQ{{4{siwAL>(g=l5m9R| zqP>D$+UWq35dNpfbg)==br@|R8&ca{ZD=u6I44!D#U7sUI20*qxTYh8%A;J9MXM{i z?vJXy42M?AMr}iJ8lhU_25{TYJ4U(UMVo-1CH1B{C1-J?1c|z+Ac>)SO%%E;Qxi%* zH`hBaRRoic3p}uCIN8mjZyk-u5=)L{YvV2?ykP|F2Xho~ zFMGVFfoa=GxaZDqRP=A2yDflqrVD99UJ{0? zHb#S^jLx%Kd^=AMEyABZp|d|J@x1NR3;kkptZ->A_FIDI(;upTafX`EDSGQteEX$w zwE3)lL+cW>ro~ikl3>KFp*`W-{!K{J#vSkNz<_?+wl`(EjssmLdYJ5vkB5@3taG^tnjsGC_j0c=Xu{q!cB;AlZDX%- z#bY@OlbfV*yc^6<^ZY!ClWDnYqr&>D6+ob z`jRQNW|(=E#P8FE*!zcuP3pFy^)Pp6U>PF05Q~5fpM}`4NBK^g39w6x*b9Dx+Ty$R zoO!Avhb~zvP9^dUs_2 zGpE#K&6dPV?l5Dh0hP5?_KUdYioTWi{fSfu52YPX@>@MGyM3sJXlYX?iOnb}2X|AF z0H5JPs|M|9h^KN^1%|^grWUZHD*=mcC@GIQ>$6vAY^Ja1a}baQ7CxSH0(3f2tAB5N z``Ybnl~Hicu7zHkLa7RFzq3l5aiPTfV>s}NbX*Dr#vnif|rc=R`Ku4x@?d*h_Z zfu+)aEwdtPo*r6Y^nO&{Qr8Z6ap?Pbo)b~_3#hFY1S580$sVMw@KrKDnq)Lvq?*v9 zJiR%cYZME z(7j^ZMes?JAhpr`>jz}i><^2mStX6ASr*Y{OdaBnQA{sb+2EtYMM560Y%XzISFW_- zQxik%lH)eZ%xb&WM>aF1%`rVHHgC-2O&#qPMt47GmWSxq-7xYe<2Ch$&YU1}Ur{hEA946~T}KCR=xn|`8)oX^RDb?5@t)@4F%pVla`YFoufZw^}&+nrk0Kx@c$80&Xh zmSOu0774+pzj|t8l$Yvz-1c`CJE9KCqp@A{*)_}{AOqgd$S|T0VlnR>f#@;wzJ45h z__l@cq}B-OIven`fmt&~knJC5B)ppGU{Z&anzX#^}sgT0`PqyRGQ=& zA(5IOBe?>V>nh8yiKOp5g6Fy`CGAU;rRr%R@IoZ_ZN$UE)eAu|v`~_3P4_lJS1FAx zRMo;%XW|9XR^$05JYUy9*NBZGv;B{ueMF?Sc!n4pu}wOd(U7Q2Y*ue^BX%&%Ya}u7 zxyzzp@Z1OfV}tIlK%=losnVP`Ii%hiwx25AzX3k}3%@#iq0J+pm(EJ)T?ewEH9Y`dUq)l(tJ#l=O>nP zE1j#Zc@IX_(pAH6Gg!pvunqm^S*Fj$H4jb0#*juOV&`@he?U*IRD@5QEm>9H+VjQt zt-1cF8KBzS*+IGE-9Rb1bl({CPLlbg-|$u`JyBh+$xdHAvtH9=sl$wE?N7UHr(zW% z2l|-?LL#hNPnqAGk?6E(R|a#C@AfrRKfxe%My9sYo9GBnak{w{zFF9JLA7Mxyz$tY zzcC?0JEH2ka2F#9n`agPo|oAw>nwR6SnGizm`2677${#N^(PA+8*{O`-R`V@t&q<>wjZF#0OOaHoZf1=m-6{3l2|IAB3B4FaXr=g>t$=v~h>NZNj{)6&W6hh! z;=w@giG+k*8!)xTA!hS$>9MF3g9$wXEqjYoyeqs@t@SKCiLY#1sq+HwBnfj>L$_Jk z1CfN3Rua52Q8Dc1edX83mGl~?vix^}Ae_*f>stQMc5^6_7`;Q!lfS~gl~B4x=`-6F zAidr*fqqJh-taNy!ohDTvC;IKkRaQ546p8>W`B$3`hH2tjkP9Jv3}}Ym5{KX@ZSn>v~2KdWGkMyC6hhFA~Y}B~f1%i=5=gX0~8n8dW)m$+?y@;D9;~ z+IfJ;Hzd2Ka9n?yc`DqlrJ3M7bsqAeok;(%9XZ3zaeh3%w742rOInZnj=%g@=M(#59*lzD{D$g|d%R{Fi}}9_{Zm=Z2nf6G#5Dhh ztWW*^24HT!Am?d&Sr5>K3w7uB6hP)RAANZ5KFa$C&f5Gnc83VYtvb3&8?VORn+{44|H2zzBA|V|C9cZv^?fh&~Xa6XMGfVA z9hz(2TRD+@LI)7i>M)@sKbWs&=rdUq5p2=h{2oUklfnC*aS;f9k(s9L18X$&IVTy; zrBn9smI{Vcb8UP2g!RhPZ?*MMO`#+owWy;Af%Nz+R|pa~#Y9wv{>*5>GxIY^pl^_F zaYU&yxfh;nPd_VD(zLX?;LH1w=AIHv*l#GK+VFf2io1n68z7iTH zy&I4$rnHg{fs#IOPL`)*n)g;M8?O}T3{8|pn}QU^D&+c>LU&F>+gqbehT6ptzXSK2 z;v0ABO1FexT`cp%OQL_O!k=5K^}8rdGFw!GR?uLw&TLu3->IEat6|=?Sg*rFK1rK> z0AH!~V}>{Kdhdd)gHAKsq1&7K~sL?P3+(Lzo_slI>@&{ zBzpzuJW%DIuG0bbfPZ&L@H-(WoY*SC-uc`vs>=Nx0HJjU5y?IaW}SkQmi8J2Xgd^z&qos}YUeBKW?s2m``AY&!?R3GS!>hGK*z}e$o zEtlEnj=E!>njmAI0-c6CK2$3+&n@xOC4b2D)7^bVk9uHUlFGq=wP9YO=B+T|32~3p zv$NsfJs2K2tB5N8{1W^nvzhM2Z71`hba)RK1^Bnb%lP&Wfk(%t99lsh)Vd~5w8cs} zxDL1)LLT&NAQbdmCqd-BAy4$VREX%Kg7jY?7j&P?Mjs z5vZidtKZI<_3MhZ#RjdiVcfkx58uRsV6n0`GjdS(Hdq$Jo;j%Ye-e5Er^JwNV+M`w z)NyFll~F^*mqGuQdeArff2E!s4I-&D_CLg)-b7akB{8nivPjZl3=}#oLG(URZ80U0 zS|fN&d%4accbjU@xAL5Ed+#hNabh39&LB*Ra{Wkrwi@3>E4l1|rM}5O`ef-IyqwP; zw1&R|8<93R!7GJtr!|0^kT$UKDoO3LW8jm)Tj$Tdqj+#XR1{uki+=b6o|05KV2M8s zpzrj_eBR!ff2Bd?Xntd>2~L9hyi*z=Gg>m2+2|x!wb|z;ka_0|`p7>nvw4X7RGnj8 z=Dx`M7F<#iZM6RVyN#Kv6#)T4aMqT_)4}=upxtjleuVh#mZa_R>;EnOY`y-s_Oi*S!0(JJ=>Dq?M@KkdGadVXUrC0%)=TM0MWdo#dC~&Z6SdyOUC_3q zTvDdEH$@vBo5xwRPq|HZ8HHT7#Gwp(oAN~z`;T~)CkIrpjG=bquyNb*G^Da_#_t$a zO6l6Qd*X_8N=^x~bjo?sZPJy>crPcAELoI^=Z#C+i)!jGM_Cr(?KrOX5=j}Z_6AKj zZl;5lwnt+{mbNEpl1rNl2~y>oD+v+HTN^);l-v8vsHzoXKw6Q z==Zlelo4#poQo1%Ra)eE5Ce(GZ`xFM*q0ZO-95_FZnr_;@+hBVp2_$j2f=guXzR|X zkX<_lF5;Hoq^+EIT->;rC8b7^a2_`@4Cdrb`sZ4tOo-=4{NhOoVS0Y<`9a9-Wb-`D zPaB?ba5Njf^l{ZK|HO#0?chWnl^=GS7vgRD53~|(spHlXZMAb%5^atb7ZPp8AGDHf zrTx~DZB~a>lB*4_9g2_2PC^gXQmX|FE~UC`iQnZroXx4K?LXw{s_ehbDz{#8=4;nq zYUZDpncL-N{76x@nX<>}CjV5qBQPP>n)F$Ksgm@~v;YxCAxsCF1rw*LgpP%oO%e4q z$KapzMYW%Y8!d+m-TS~u2ctxpCk-VRD4vsxT4RB}EhHW7U*Cjz>-fbJzo2DaVBq7z z8|+x(ClnW$=q4&qSdwD(aZwVhBvcgnW|A=^vS>i#2*4oZO_s9j_&b5sECkujkeFhCa67!$%bubG05_vK|&YR+6-=Y8s+n??Wiyt@?e~GhvjH4 zm0Ie#Qy_R)d&rE|Aj#zgB6uGHr+6;tx3hMiypKMAjsV3p)I#32w)g{l@Fk2G;`#m> zxa|H9D5`Fa30y(>xUEwqD5_f=uHf1Q31AMWY5b%DkcX!|HNU(34R=`><*0Gf9Ui%& z7+CIEJHPSnN^)=*LoEkQe!cNmcu#RWTKi7~=3);{uOEMoHtHr2nD6At!Iq}DDK@=pkssupQq`0Y8s^g7#Z6cq za?2oPc*F~HBHd?**qEQ>N2MGPCfh=Sk2AZxT9EHef(=kQZOvq;ZpSVMz3 z)<=ySG4Gs(!JzZv)bAk$B9=yj5p<%6Vg?f0uf!#U?l zNiyOCG^PQG6gQZ)LSrqD;V&p?tB@5Dcw=jf;CuT=g8pa_;vAdU<|mGARCH#XH&B zi>$6ccKSq1lQ8qcY2n}7oiDG)aKOjGH0L^WAzQOFoBJ8O`IG@k<|6I2N0 zB^QwgSTsdQa3r#S=(wZVFSwYI_upG9!-ACStJ5;b%m%S1sp{66`rl$n^ei+f$8DEz z+ss3uWtX`dNGy^jG|9!3=87C!7o`2=OlofZ`QnGxT)AonX#S(G6)y&TlO{jcd@l-t z9TawEJ$g8Qzb$y94d(pzZn7+k-~k!e-p^_Cp{W1Dop{1iWF^w0)B~O-}tSn zsudaRjq0_vG};!PYKkZOyV*K4{D=2~Il&XOJAe7^j+ArCKhOqGZws3S1=jNt4$8P(l63-AN zP;z6~FJakq`1I(8gq6N_5acvu3jd1pv-oxGla`~&7$N>_jT6w93o0k)eqHDQH+z%j zW#v?fa=gHECHOXojh?V^v_2eWw)~I4{Zjej)Xdg4=KoaUU!r+w&h;Fd$X2V?9hS&v z`Kpp(jq0PBs;K#oJoh}OY%~+~vh6xIdK2}gUzRTCMt>ZqFS8$Itvc2(oE|HuBt|$l zL6?k}`BCjhA1jl3SoGTP-1ctty*s{0h(iUUwGpSOc5o~Zd=lQ-kGP@er4#x(9-;2+ zm_r2zaV6P_J1^w7Gi?@I%n^^J(kPpgiuqEnt$?QG@-`bX=eK~IxRUX{4%e1;8jU!* z@av~Jg>ce&NnJw)vn!0K)kZ2Z{k{b~Ug#InvR2v8UeL!KgRNNSJd(!{dS{=rx$s-&88;&whGy zH`jwnWyk20zXTqs;NJj>Dy3SE`pOdbZCRLAaDAO?VY5L6xi=BxO1ZCAW+I|d)6S|o zc`ejpTnO;h&_JnyxX6$?gB6M0D8e71MhGE>7r&4OX0vlR7s8#Sk0d;-4=!sP;4SQm zx9N_giTS2Jh$GWYU7)(%ztQ}1m61l|oC%=eh-gnfC-_1%owVQtqJlR#vW7|K4%VQ2 z*N#!u?fwGel}XYhD9A)w7@;tkHXiJc6E)6pi+F0(09e31bDXfXO6zW)_*-S~S zrq!o~?0WKM7-8pN=+D=?f@)S5m116TBpDPTShoWMHdL zs3Cjv&x6h%x6OfG1-75aoGA*(K#$_fK$}bMa5P&fN%DVMe-kr9wkE@^(sZ?ChhfHD z{H07@wtG@J8r?gq5$XCT1ozgP;da+{&v%b$YY7{}|2PIVf@|bz$2QUL=~rMcLfu@T z`;)7xypYK1-KczA8yD7yaRgDmjlwmEOGE-Rzb&R^pl+<2a-eS0_~J*kT9{2N6di0%eiwDkSRW$Fd2ufNNwv3gB9-ER{b6uAE;>uy&Iirhup{i zO`}{u=Xu98R&%dAm#+rK?ut^Oz=~)eP{UEHAu#DTmMwPFyLE0#vrT){>lguYfqxS} znxZ7rZGKxs3Mf|4fZ<<_Li57d+=H9%VDCINT9YGc5(>ki!^z1zxD1c-ymH$x1g2Rj+N--_(N6&{SRTcBTzvdXCcXTyIGEYJxo07(f_qV)vRra8;}~)=W{z82 ztw*wW5<>`eF1~tt3nUW9;n9#DLUi(n#*QI^7cWIBj450_26kN9wS-j@*?*EO!QsD1 zHr4Jg$);1i5K;jh!t4bVOdDEtnJtKUM&_3v${R)=4<3$_PY+^#FKq+UIS^WE^~d!` zP(hVsUBzrNjt~e?SD`+NjW>z9@d#Y$s^xa7LaCYFBU5gBV8F!ATQS8n(J+41D4p@o zE!2^6Cve}Qx3EpZ$u{N8FT|R>&!e6qXMtNMfe4B1bq>`z;9qh!V8@pq z-Kf0~!ylnkG>_rt^wq^dV!w`sB}e(Q3rt8atj}~h=o9~JGM2(<*EuS(?ZFZl(cfwG z52QVH0U#}9%U`55BMYrMNyrkat`BGhU!>PSU#{?OucX7_#Ep*6{xnlPnqy2%+ZL5U zDk-1l#LJfGNP-d}dQC+Yn5v(Y?Vk^GSIh*hJ$dH%!zZ1aBygfLku3%cQF3Z4YLN`= zYy_-PVY+cNK`Ygn2`H;IDt2_1Zz@wA!%$QDww5`2VxY*c|fjDx(bY+XEH)8wD%9A~!UhZoopS|05aP4vST<=MY8!A73B;W~=eEw3ikJ)dTvf!qoXB<(?P`#2XG~<%9HBuVC*Oh77ai2qy5P zQo=%1^yJYY?MryWNT@yJc!t%>=hUk`)ZRROKn}DU=^h{TG?-vQ#}?KgP?xWS$OWkj zCwhf$;QN?2$E#cq2Y{&;5pxVGqB5P*M%cy916apAclhIa(;5gT|55@OJm#P90wxxE zdN?)pJuV;K@+?n@+%Hlt=WU)}5+3)7Fh67PR||OD8^xv_tXA#sXg);rHtQM?Z`6f^ zzadH0&mh(*D+#AOAB;s;f4A|OWe?FC(vcu73VpZ*NtZVC>`G4&UIx2@zR-NhW)OIklbPVz~Og zaIRhca2S(B6pERMGxGPXSE7(k0)WAh!H)9;vgxzkw@m>Gj7xMav;iimw`8u6JG3Wi zI~RRba#70WVFu3Po5&LbN0?SIuvCM=eufTC|I2|b4e0)}diVl7O>F2wrNS<1-Ea5; zyJdp31-c~#8nlTelwrsYYtn}Za#zc^oJN?bP#1PnRhL9&;wkQLnT?Hl2M5gp-wdy2 zIM6T;>EIc?$$JxwW-$`_?4KpQ2I2kq6)3O!)f-}r1cmJa`dC4UkZsMdgd{ONr>`4F z;WPNip0w2xQJ&``6Gc6~@7T)si9nFHhO{87%!w=|roE=%ETH5b_-Vi%DCXU!U2v&3 z3b{Juy2uaq{W0ll0Z~Z` zJL#JwX@de@R$C_L|*J>09uAw+;~EbGYo7S=nuoJO}-NB^KZv!cV@Pf$1IiJH{w>_G=5_ z){{dL2Dp6-KrI_;hi?!c=1M5~owqY+MGAf=Mc{+ zX!np}IYTS%)1@)bm?A0>d4rhKVG~I**DYL#zW76U6h=fq^5*`&B z*Ua-sWRvE_2r{P?(tZFd4d1`H)L;dlkv$NsS?HV?vc4I1;Ko<*DA=0smR*)e$m3uZ z_Ms`5Z!H3L+aB|xgI?J$2#QcN=SPDSaf95cyw)a1m8+J{=)8&5zF zG#A#y9M)LtJvM*$vixVe18Z5|vC!7q#d+)NFVXyvw4D& z_Q{(FGF{n60(?z?H+z-F2ZwBiyHESC*^E8vzq1(`^G$~Tn9X3_HJAuGhfe^wL|tdL z>d7WqjvaeF$tIJZ%a@tX?*9#Fo}L{M)MI9W6h?{G8&p4+JP(}@!dN!{0W^pKxNU^Z zPdgcg0GFr}#{oYOqj+pb$0P8C9b1^l5S|PdY0J6PRu)MgP}p!zOkw3R!a)Q%ehtS)BK8)dxy}ga`QyB?dIA}&+*UOX8IId$5NX(X8)D;msRGd%53fTkbC?dn>D`zvTb%hC5 zJT~q4*I}m@U_K*jqR6u9YrU}%@@_F~k+tC!)+zm^uFz0%RXB|i%L@?)uv)*#^Ar+( z|1k(ycEaa>BAn;lzl5Vf_vipFTvKKBKNF5FfN)_FFXAxEZaA3$cm*lv6K9wja+<|qNY4_%v-oJLKgOfKSga@ z_0GxLCi#8jQV`Ar9#u;C3k^N$FX0Rt|0Ntno>lz!)GK#%-yEQ;l*C`6JYgN;dp;~F z8G|Cl>a&#mS)k+)1uhX%o)<4II@vxf#4%^jYKOlIK0X5v;=O&S;J>Y1*dEQ=_^khV ze8QEzf&ShDiqpuC`66P*i2>-`U>1|(HlPTv!b|JHwKD~C$iBSxEJHZ++=I_^g}$W3 zlAI)U%*%XkS9tS%Do@FMzFl8mr>6qNgziYdXA*LPk%$)LUgZ(Rz+TLRUR#7^5lZrA zyM3@w+bQmZGvV|4N$CycPM{>U-E#;Oe!MJvywGj?faLVAFsVw|&IHx;Zvo}%JU7B_ zt8%1MN`xO8N9ls*W<6y$%S=So-D7z1$$)Bis6ehovs5p(s%8fbHx`M9v< z>w<JN26wIRh4uXF13ppURl}rJPXC!0yJF-5kNq*$;p3E0ngTcK6xlo(8t?HT?SS zRmE!wU#=|eAn$gGvy|YT>_FkCCtw-Cv-f4 ztb?pA-->mB&=9bOT596(dDAf3C6&mwY)0& z=0gVt#JVG?-QWXZ@(aPu;D}#qvi_qTTG<%WeB_c}^T>kto!t$8QwscKY{a(`MVLF>^7Z_|)p5%>?shH9^WP>35fE6_&PzK5nzsmwJ||O?XS;jNhtmv})r? zzANSG;W!s2!uTNpj+4wo#icdvpyfJ0nd7#YDnUKErV zU2Zsuqyli1?<^2wG@2Hr2R4=U98lP*)pxMV`0{m*RmtN(+GYH@mZ$uOqj&X+D9v{N zwNN2OdQEHDiqrSKf<z$8|Rj4q@mxBkKqD%5B7&hwFhEuB7>$`fV@~vn(UYr^*)zy|XiB!=3 zkrri~9ogmmBFFZ%%U6iUKCfpixXV620wexilz8wcz$j!8@vS!BpbyQVi3eCPQ4hk* zU&K8)>pmwL%aFKh=rTN0Tk8mi?MpIuA>2G)^L02|bEZym|28Z|wrL$N9CM~b(|pAn z${+sIdq|=oDcFdP7ltZf#TJSFC~7Y5%ycD;DV*LH!_9TTIKj~H4?LD@FT+uyqVQr8g4;W zbj?;r!%E^tSt&Y8x3`ChsJ`GCdtnDR+2-X(~XDEQ_Gn=4RzovGeNos)o)r zMn-+C%kz5n)Gaj2zc_S|Oxig~if;Qo?)| z(90Em^4rJu$g~Lx1UHOfrQi$qKHmh}q3XUxpb)AtLEiS5{T1~$BTQF{&_vGNHdbO% zNubdI@F3E5yvUqN$6Z$Sc4_J+9B@F|Un8TM3TgPhPCge*P@*mAp6|Z>>+B|VlN|G( z1OgMMBX4x5&A{tsneVa^+B4O_r|$&!B++|m<4kzHtgaQ6f6YtLg8V#UwWwB)qTuK2_BLCp)(lF-rq-zAioq zI9(&TPwkm_{&<8Y_M6u1($MWv<2dK)vBcKu?b1+^wq}rMMAUr+uk*tQzy@a;+#LXF z)(kxm;f}?S(cP6*d)Ex&iu}6>>k)Z#{b_Zi+B_N)R|t6R#L`WT0T+SN(CUdX5rVm2 z@Eeeb4(MhfIe5x8p2Pc1E~RTay>ndNY5ZGoFuFiLSo}zq7uzj^c!q5|O+KjoC<9jN z2@_(V(dt5B`cK=xTIDpozpd?p|3X^sWw|nW+uT>VYc8ysmR9PxhMcYp5!7+2-ZnRN z|Ha)#GWz3gGyV{w3pql+jywi5%Dp2v5qe?f46iGBdp&UBejqXiBS%46bI-FEeV$i! zvKjop$*^ylEiXF18|9b=|63UrBYDUF|HxW$h9?*utRA~q1{j%LHeA)Y- zt#b7D|Jo|Y3HUdya;5Qq@RkSGbWQYAmOBH9f&AJNUtOjbLOMIhHGJ}!BbtxCW>-M* zeD|x5miq@d%1>ZYjM>h=Jnx;60lrT+cA6HQ+Mg0n>NCfNwj|=h6M*^x5PWYOmtVM8 zx&^~@eTY=r9~o|WfLOcB`T}4f|Gh424$x)e{pbG4T4)ov`swTl&E49seQyU(JGk&Eqq>3Zs>_5RSK$0N`Ca86P2pHKR{^Ne@{df8@`5*3g^LO5q zNBn=%m&bq57xTQ7zTdI*H$W_%T_@~MEIqqVPNcSY%lP;wWSuK|J0O@I2rVk{jVerg z{D2PpcWXTNDb6x9`q@g#7Zb^%n~W?&L9RAEokMK=fPAHgPSpY0{b)O?=XitIX1Ykc zv9E>VVYI|f$Hk#4Bh1{sUQL#`q{FUWZ89VOaWoZ8qQlONp%Vwf&|?|KKfpU;glxU7>71ES!g~ zUYMk}h|I~4^4Z1dO{)%>0<01a{;k>T3z|QjobsXAK4`2Hv7$3^D?k9=k>$GJQYJ3c z3BFxt-{z|K^o>a%ZCdpfZ!*0-7XD06M)#{`umO7sfc={rl@<3^HDYx*Er}U&#B$Bq zxfOl{BrQVRB(>zN{VPk6=lLf~`7lvvFM9d-#nr7_0C6g0TvY)9D!336PR)J5)~6tY z&$;-_q_aPRj%ovj!z=(BEZz7#?Gf{W1q1JQE#q1r^{s)y^L$&Z7mvOLKRk2Kr8`Ea;1iL zfa_>u)?pN0ADH{DT^_O9_*w7sc%qv*Y}w&P+%bAzVjfx!{*dV%ZsZ zG5K1pz*!pPAk=0Q{Nc6?KAb&OggxYVoJX>89emYDQ_aH$QlT-4dNJd^I`#4%q-dTo z_Bo>|M@Qy!TqASSseo`|cJ1aQs2jJEwXq~=lPPzdG-h!{R{&Aem`T`Jf2R+35T{tb zvPSB9kw;p`x5_+uacRMJ4>RTVN$^EJ`g`%=do~b^VD>ll@2b%}w*aC(=zH=*>+aec z-eU{Ph5aH2z#Qz(@*bVPE_Yjc|0Ij}xMF+nn%e!)=~{Td45xn&xQPzF)k1V)wFt$G zT5F-=E&?qXn0Bki;uTMK=5`J^QU;pyTDAL(J3l=tUkQ71-?T`3a!~!Tc;&CVa}EB> z@Ll-IcM|fHR~{e@y60FsG2cf0sO}8kAJJ-D#~*W@&)kA@{1FqLn(qfUhM!oGanypq zr4JcfvHH+12FNufZ9vqo8m}p174Fk;#5-xm+drgHN>n;a&56kF8rO!2uop8qf+e)| zToXAD#tASahJwCY!d`j@NtT`PY_LRx-9(87kYLCUAQcq*h<2O!U=D>v)7b4!XT(%OI_W6XQtP3 zp2ViXXt+d`o#jqOiSnO&MgSeNoxnBwW$j zr^^GX8xw7sxAd0Tszid}N5wf+Rp~_KN{#Gs)?wMmvmtyOvF)=VOnkT?q7(vH>Z~2t z-Uqm#=iG7#|JwdmaFFjL;@1R`M=f06Vg$sIil1bJ@~LH_a|96#c7IHzk7f`7zv<7Y zDEbm_rYK!nyxN5ZKI+TPRfK*|PMvVxZTey$Jon}S`stQ`G98vXk@y#%ZaKkIf`_OD;34-GpC&la;-cWzu3 zE`cVk@Z+i=AChVyFYTKVIeb+V?)a~hvl~2BO%3cxr-$>5eFR7*(3b;48CML8`ef&D(rx)N;;rRY`yqbmpB~| zuQ7M#WW6`v(%VdC0URG|u;2F;5hh-8bz?XB7avs|jPa-ez*=F!sS8g9SSx#eAYX3s zaG!l4ILX7EPcZS}4|K6_zbSQFp#=Z~ycLuG@K*fI+`6j?Hv?7}2BBg;V}U;Fu~rfW zZwsTb#!OI&(uT@k*HNWXOD53`ULhfGMv2NVaou3k=#nR{!TIMRz0k-mCqVx91jt$u z7M|&RtW0V~Yu3=0=3URL1%dODuwM!KKD7uhfYPvxQ7ZgSTD$5Jp~AUWt&s$SJgRql z^hit+QPwqI#CLrYd$1KeK=aoy%zt#a(vLbR*lVn4tt@MFAG4RA&Pr^US4 zpL*|39Eb?8YfPoW^OjZh!^awKiq(KH>=0C$#UN@FDY488Y)!JS2pLODPjjH7W zG96+6k?HupIU_3fvY6Dx*SJ5+#GwzcKrUL^h`=U#DQog}Mf2RDM z?`UA`2EFMDFa5TDc$T?e{_;C93x;UhV_5sM7{R|EYp9IuA2tYJ@ZY2mPOsH<;!k%~ zQ2SR5Nw0MY#xrGv5Qor(`>}iN--*Rrlk@7C)6ajYuV$=v{P+6mBKO_e8v#4V}Eg;?T>^Sa;4gIk~ z(#ig}R>d8q{J(b=K=Ry5?{&qGXCuQ4p)LE(7j0BHh!{X+Uf zvrS~>+J8nn;$}X6M?0RjQQ>`mJ0ZOP-U(?&w;Nq{i1C6!03xO1$g${ICIW`Qx5L;( zpt7GvgXVGLXASsoxmlrqlbdbmx1O7oC4Xh;Hf@EmF~8c<+tu~iADnk)SpJC=eDm`B z+v@0`aDt%o32U)Fy1j9Z+b90s&9Eb^=YiMQ2@huzLs#WXbaC^WGSr}hSHTZ#MjD1- zZA-yUhl0pP4P-BWAi@PdF*9h2-nxoI9*ih#g}((ube~jB|3oiE`qRNL8f?=MhwA(! zt^I}Ln7k#dz}|(ZFi%-T_iv=Zau|suW}}QkI;sz|5eR-a*URUZ54(()3uLFy7Rl&L z{qf4L&;Mw%Vk9}5u4evevqrh+iz58BS|wDNMK*T^Sgl$GtX8=eg@^uGt+I#A8Fs#Z zg?+>S#wNFq{vEniK`@MxrZ6VF#Uptg6RDq!My0phdz>8>+DH7;>YoCtp)k|RR`!@o zEtFRIF>3Qf#U30W)sfu!LfG&0RFoe@rqXs5eW|W!dneVYMRi|(>TzC6M+u$I9tsiW zx1Erqh$wsPf+i?9@EH=lLr&QVsA=>$-|j=;H7mb&^=H0N!aPUG_f87dwA=kk3Sv-O z%;!JU;z@Au2R@LHjsqk-2b>lBIf)HoRRmA}>j-oaf!%OlCkV)~o>#Um(V>gNFp|5c zWxM1@J6n{1&>p&)23Go-EVJc@D8z0zn~DF(mvxFDjy~CC1VVzYPPXy;tsA-LRlaxW z_vokO=kFPBP=}jHTIbj^>$4C;A2LhCx>i#{Eg?w%8!7+n@>nX)F-k7l zXM>X;Sm(2>vr}aX-Me&E`mZUoY&C5XRaOl;F`0?r*)n9=%do0Ucn?4%4t=dOeZ@nX zX@^U}5huHf_nkerY~gU1XMuaz3Q;SRPy<22x&452gXg({Cb=H^d9RPK+wh)IV@|n7 z8f>INdM4F6MP1>OtglU?a`lWE5{zp0Z7p!E;asyT5Id64R&aI|Vel1mW%=wE8b$Zt z6IrYf9J|T{C|wI_rW83u1iRE~O+S_aLsR2{&BiJ^VWyz4CHtTAu?;`B-r^}k;+E8s zj`GZxp3G|V*@L~W{L+=kI83z`h*rF;8T}FjWXP^7M76lwN%Yu0#*<^f^O;~xth;dv zKrx)~<;EU0E-d&WPBWD3XPT7LKJj_6&M)pO@(s7VY#hBg@}% z5AJgeO$W=71pTTIvX1A&(zeq{$fFw}_EtdiLu*Lo4sS$>Jz=x$(05d?*3#U~XJ zpL}r?&ywPF9}8tBMgZUeQG5Z&WL+J`7fH3Z+7)YcPbTaHB^ge4CA(?$>2fzK?dfu8 z9o2C4fJ-x0e6?b2&wwAeI8-7l-SCjy@Sye-bqeP!{pq`cwJZ19InJMc@43Q2)_`oB zpZT@^ZVaBI>s(6ndj&i55P1qxzd~`<_r)cqF;Lz9hVh$Q!(uyy^kx-KosD`id^`dA zt`zIPH=Bx@8_V~{CagVV`P-skvdwEX?>widqvp(Y$YrAhH2cJw$yzN8dGoY>ID5KO zT`fLjGxyg-DTtpkx%=5ZK@wUi3fQ`NoZhrd-aPYB03jo~od{|-R!y6flN%gkn8v^`2s_>7Phidre9AUoW7=$zE=a@)pyaPco*A&?wYlzF=yb@LY&LRF92OymA zyHkbvl~Rvs(l8n$sqam_am)nymB<%Q(r81moMgJ@f@J#SA<9Pw8k_n2&GXy(FD#EM zqCqKVp29V>l1&Mx&!~&df$CUwtZPy7SY*Kq?9N=B$Ix6K28u(_6K&z&yhlzf_Uyqv z?6a#@!=nQ{sUu(HElL6jn#{{u=w7bA_m29q$25;X?K@tv5iC&0hq1S}VzkHobkAIB zyD6G{ZkbC|QTeq#UAFNQzjo1U&}_nHxH=nf&7!@p$G;;>3SD(>rM%Zux!+Q`UBkDR zt!8AA-dy~-t#E9xDejGmv!3=mT4#`ZaV1;i!${+K5HfmDTQ_KSjV}E1ip-#-w@SM( zy1{GT?0pX{3rps0!2DN#diJpCDT3}EMCGAR{{9snLMcM1aFz60T8r;en>PjEr_`hwKijkzM}yU&DIJa)~m=GiqS%&(t|8_)o;& zzEoF_Soj0j*KRIYgVzWn`*n#}d3D1w+w#(m}Op{PY?XbxR5|s)`p65QtW;s_V`3!-*c$Ge| z57taa&D%r}g*1(jDx{szFaLhXk%<&JNS&E9jc5jTBqQHQ{Lp?HR^mAi3;6_sRhTJ0 z)<~R1_?8J1d6E#ybDUKKSeHrUmQlq&uA#t4f^8)3yj3o4qnk>??rnqFPSQ=tIDfVJ zb1hv7e4QeJZUI}lB=g{BxIAJll=)yeSIHtw)=4o~9~3N8CC~J|U$jvwFv+B)l8OEI z9OjAhm~F=u1lO-_&PyK%jGDBG0{5j;WI64UKV0a)Cu(0r{s^;GQ{JtZ2ZB#UR)$bb zg(@b-fslFbaAcBh_$HjtTR9)BRRUM{16jAgDM=o>ND)})bTt=wLwQh2Aw2#z;!&pO z&wQH>JD2*mnGZlVFgW-_njFY+!25&QUeTV&6$nCEeBh1x>u2ACYQGA)hcfb<IT zV0zQoLyc6*LNDSuybJ`V)(iHLY|PD#4HKo-=soISSrRXJDvwfzUb7=p(z8{o{IY+- zcEAP4b}*SAPI;0Ey`q$>)__>`gU6M8A%-5KEYJ&3^hc~FY?dBxP!#7NQG|oDpW&() z0Xs8;J7qAS%ilIhE>>~%DO;U3R#E6rhBYZfPA)<>JB*WzVNO={&tl$fKx9d-4E&)q z!X=(2MJ+U^t;@@ijjcQ}fJSIeSGGL|ahf-yBCvv011+wVpheZJLpDD#kQ5$Qh`;g` z3|KDkEa$pBKP*+nXAo>fG9+V+O7P&k1nNuSY7`ox8?_Qt0gOI$3|I1|03v-NZO6$SwF_VzZKUp~Wzb(bn+_HI~s>pdFUOv%ld414%NqNvY zcCGm^U96YkaMo3n`FJJct3H0T-wF%_468*lv>D*+0v3xZq{ZqE(U%fH_1fYqp!_8E zbu99=(0XUfmOm1lZ>90vkM5%QX4i!PO)S#j`Qf`j=wnI$Y}#XL>|(tfb(txF%&W<& z81Y+w7B*d;?Kjrfyp?Pre7W5kph{$&dDQ8MoI)SN=t3W}09G(%lDGMZl?328Z!MCi z)Gt%OJ)d2z%0YL?fFA9j;C74)SOes~vj!|7>I%Dr-YdCVnZQ;t{?f;nJLt*%)~+ny z=dzOb%_Fip=;p8f^|xPuC4k#~mod!mXU_Y}$@QWFa8NH2RTM^7D%-jPG0Iiwt z7g!C0Db8#86G@kx*{+b2(s+E|%#Q{}s~1_qgkMCWes%lf#?SZ#vh(2D zm7smLB~KOFg0`Z7`F@@C#{s3rqqqwJ2fHCBjTab4tE67}#Qv9FAr&vCU3Op(jWjZ9 z1xhXP^rx>P=UAa6U&Hi)7bIFtBZ@#t{mEpG0Dvv-OGz zTjiW~Gl(b|m99o&v(!zB(oq7Fp$YKGX%aSrx+?4TO-A;K$*-}ni-YN|c&aDh5ZNg| zcd|GfF)qj-%Pek}9?dB(lH+DpWu#Ol_hms7+tAM*Ldq&`b6VC6o0F^@EyoW7v&K_Y zD9=@K>atd`%bQLLmXfezPRJ9(@M58SG*{;!W0NhON-q+Ob)2agMKJk5%NkF;z-V04^%m+DE~zM^=X0my}A?a>Hctd}bg;M;qxl6WiIa zu@ZZK_1xIccdC{eU__RLaq&f$#Wj?BHrC7`e$P5{&*w6c44@zAaMz=>Nf*rZ+hd{}K+kU31cTcik#mTW6 zs@?0-Uc=6CqA6Mue$+5F5uOVMMNk;F541Rdc1=_Y%$@Fh`wR_v(2XU`B=jj>bz2a^ z6U>X)hY5qR+w_ryU=I?y;_vd?BMMUD%VKIAtr{ zVE>uc7#0gbW)hLfGbHr+qweR`h2{EU7v-h5w;S5={IkX=desk7sS5Wf1I|$Az4?X= z!0;JY&kR{@XVn`I$arQ7j)V>+&{Vv7KMl<~I{LMS`llZuLKEDxiH+>R zk1LB!ywfcTBmmjwQG%;q6drT?2`xk!>bTRFFGCLlCc=e%9s^0?Cq%~dkei9G(zz_` zkM?U+fn6WVQFI5j_5N4Q>IHLdy>yZ*+#2@zJ0jukm$fo^5r@~z&VrH| z; zyDRR;)~@2+HX`2bV6(KCEfrt2nyrKg4)fy168QSQNc%p(U3v7;R%cp#H76_JhSOH7 zoawpCEH@Xmve4W%Z_X!Y<5-UO*1uv&k-Z_S=1LkRKo$ygaNix$T3QmA{t%e5+)a;> zfa(O6#{B|{NmYp)aBaZZ^riNp9Y`uw+7@!IL6QVVfl!pN%`8hk4#-^3_d;IGf@BKA zGXCn^r)pbD08BC*U#{?$;{L7J!3>@DY_6EUXF`Tz)aCQpP>x)F8kcm^Re(C*82RCr zE9LrE?$Yj)Sv{V?euOW9!pkn>)5)LeepHzyk(1z}@oB@UstorKbD<jxc6lqYYq+mUN96-y&ToCC%q|j&SQUUW`&8d26M|rg&-qljD4puJso;5JaxoeK@pDI;qx2U633*$=O_ zd1A)mmfF_F+@=|h`u zMz_FZFfb=vO8Zv^jf2R0lj{9t|737EY@QY`o@abrY?=}h7Ij4o*rakYHg!AU)p@-U z3O_OgxFo>Usb0MiMM;0MJJM(fDSiFw{GPZbA2LQpbwx3g>U=s(8VxCn>iwYllAbs@ zP^nH9^;ta$UKoEeDJr<6@pv9=niIk(EIZgN78koWm)kE;$vp{NLjF{aJ$j>nTSN8x z*65E0F2ivjY8D<$P^o$9*h$0If5loQz%sGU>BtK7eJn_jLtNz#`@CYF^R^tsS6|c2 zG`%VC`XKDyvJ7HPjKw_7CuWk*O>b2DM{o$$=kz`wNj-rA zphRp*5b8V9bOdV8FSyc1H?*S_1ZjokJ#T9YwM!FM?1QlwDEGG6oFhP^mrT zP|m+s2{!WCEWvPYRsJqBEg7+?V8X7B*%?uAP4>QXdtmA4FredEposF zZH$>ZeJ)e-QiDS<#li-=-9eT!v!diSBz#bm{FI7gK-7C zk!r^`xsc>sje*hniKUzc({Y3{V`ncoHZGY27@+r+Z97*AD?ZBB#Dbuf!zwwNCIE zWARJ$nJcU>)`)<2R`CA!J3FIB6&5LY4+~wdbGd!NfSp8dd!GbSN!$7H=TA)@+>9xl zIh0w4uTIQDfg$RkAfcj)Q_0QV2}LtpTdX znJomL=0g+EYH(urLB|v@|5On)`)QSDgaoJ_7AoipduA{dTil%R;=;{eijz4-Rmy~+ zf(}z!OUi=b;`e8;UJh(dH4~8~1szk`oB{A;z28snsSojhl1DUHQ(>s^O$9enjS*$I ztB7OI*)H@=brk|(cI@dM*VG4L{{3m}9S8iQ;4=AzZ{G&PCzPx5P?pKMIm;3{aKU-I z-G#VU`a>gGJ3Rce3JpYUowQMsPn`%^TuWfV-j0+sjb#+DYbOjCc^%k*WlIW7h21Pd z7VK>q11Y2f)zEx!zKe=dm~t1`EgL$Vd8MA-NJJ{E`WYxRq->cq@Xfvc0yYB8X|SIr zqj^HPUO7FGzS{F=htXR-6YR>QQODqcY1lyQK57Fb7~?`An1D1V+7r*?){ii^GVP1{ zxY*V}6uVg3G`dXv7Iz&eAwk*WtmvU>9lHfoX4j`g6IK8?p4|m<@r1iZbtnk9zKrOh zBCQ^)@wXUmh2Ot}3LM|jp|=5xVvuz{oP}~|3+WCG$F;Ga>*@p_K7d?bhwbB+z|@Fi zCU3;yIBsCsO=P$7mspDnEEVkzbqg8wa&Hw(axO1XwYp?21&FTRE)cxYE~@ZqRv{=Y*0xl&2r3t zSMEzf1jF@I6q#N-i11>0VH|VPcvELnyp#wX7|iFM)yIO_BJHlsb4Xc?a*oHojR1Dv zhCZi?%J4}n$7+`TuJNjzC4zW-q_ z&(f&rC%RRxn8epOwU4gqYdi0`=ki&|+OhtzAt*}lD^5pIG`>sgvZv*_071vnlG54u z)S^5Nx6Q56i}gTbKR&jx*uELrB8Fuy3|d8Je_t%$>dXMXnZeJO0nKD+K$-3GL}O_Q zrg&UX3A1^Z6zF?4v0Ka<*=T!5stNHl$V#|I-TYCgN*O(gbz@}}H_G(Rpc zP6Yf<1aSWu6Yr;Teq1=QJmVa13^+s^pq0A!QCGCjO+-YvhaJ{UUIL(ulrbQ^67cQf z?oWYXsW3Ztd%Dp*;g_)Ct_)Bd`fOytNHYy3$w*=F55h(X50idSr}+eK*9V~Zc)dMd zTYT%;IWyJ#kRNv7ZzDU^)>l_~J!3l#pWN|DUBa5h4Xfjax1rMX{SM{{3#`uqfQ7)S zk7C9F^|igWX;E+BD?ois5Ec^K8`XZi2el1SnsE=90Thp+pi+vSAr%NCzHYCgxwJHG zpmk;b4%lx%#X3s6%3z8JrJVv(h!;VBHk=rm^-8=bY7=+WSZnK1;7F8e`G{zR;v^O6 z;NCbIyC;#6zkFsS?%n36jX?elGTKa#gw&$}=Tel1}Lb z7KIUPk-hy6uBqE8z|g1Q;^vaYMkkc7`z~{R66gtg7hm7spZA&KQP%$i^(%0hK?w~J zk@Bz(t5>_DCmiS-!u>kA(WW>u;-!`l}K8#o@=wexOC&pLvfI-FhSSigErZhdHO zId+|RUmS&K^~|C<0qgBh*l_{ZbC83!SA0s+Yw`kJ(H7JZftEVK+sajU!sKDtcri|CGhA`I z5&UI_mbvGB@3~McvV#3+&xBu_o}dR;^rDs3YArAUc3U9wGYL&@eG9)``}_j0nzlN)(OCfEDPlF` zju9g!)Yie#PbRcaCONr4qP0CXGxu9EeGw7-YJ150_(BZ^{fP?@rX6x6GV9BM&2!9ePpIF}F$MXc?8b9Nu&Yt`9x)_F`F%A_B*@57em zK$p|bz}X>7-6C10g#A^(i&Pb?iDegzt@M=W6kyX!B?)bn7B*(U*?BG2l^0sT*)4gj zN@D{cfYO;lLb(kf?9;PIf-)fa(SE)yHe5I(m>et3`+&c!W8b?{zq*1yS|FaUEC;p< z@~O^OBGAgyG^*dn_Ns3*vZmkI{^>I$QKD~W(jJ=0&iq!O%gNb%nv>j6fNEBtwU1Qw7Iv4n>w_cb##Oh8cSw3oajMZU zvtieP3OW<)0>_4qXGMnvCtZXx%NU2eb85nQY39=>yWrcmoY?gE87jdb&4GF~KWT*T z?Xw;R?=iI8>VI$WI-I`Sn7A}l0h{Z4#*gA!gz~^xlP7fDPlh>uPG6Xi025FIni~m} zE%e2lJ8=;L*}a}P)y0MsC=x*1a47FII5-2xS*fbsN2t*vC>da?RhA`k0#RMr7!=Tu zG=fcla^9;qut*pbO+_78rhH(T71cOkxBPv<)FMHFfo(KJrQ|b|Gx;8;UyWy2bnaE~ z#ttRw{NUK$jsY1Q7jVf`aA~BkOw;Kxk^}~a%C3=(Qq=S|xeh~fNz(Fmq2{SfzYI)G zy5sT=2{LS+{c&0!yc>BmS}s0zfToZsVyR$psvi9OP`Q3UDMV4N436K-P@u5j@~0eE zAUJZ%a0zc%oTvIqUpQP^GG-f=9jonP-@Isys5HxVN5A}JaMDD1>=;7v9+5^x5>lSP5q^j2^-!jgn>pZtI}API0%Ufq~_#5N6gry{>v8z@t!^LlCg z&zQVSdi4RbJ_;bB%soCcDJrRQRK(2E(@#=9%L~h)RyLZ4x79kA`Zygg7T<&raR9FZ z`>PU&M!L|VcMvcnpOC_va=BnRnIU)$`8u3FYskY5v1(mKlPsx<#J8LTUqx_9ksg7P z6cyM49+wh)NjTprc$0|I$}Z3t$nzk*k}cjBiwianJCivpqa-*3y}61_3uH5F`@7n# zX#oly26lQi6r6QssgBTQQzEB2rmI+P5lw8aKv1KK7kQC@n649Z}~Xvd0C$p{iRrR2~h z$}szu(@m^HcB+&!*du!NvN=B6_0XMz@j)su4z#DiOzMmn2{ckPEi@DEg^33}(=6J+ ziDzKpxmeH*zAA&nS(VIanuPQ9<`P%JOK)oHS{olKNya2tJHsN4CVL>>M<@= zsT`X1G|*+o!4&C|^|F_Y1xaK2XJd1AnyfD+ReW0bWa{!kBjGt!OztPBzn>VqO0QDF zMC4sc;WN(}D42h3(yamaw7k#abK5@A|JhIj;(4$8Rjz_rQeD{cynDnvWh=}wl$T`^ zzF|SKoPO1})8qviLF~n3!@ztom(8i_Yq?`d&M(0N$EDUza2?k3R%Y;`nC?pdLI#@w zsJ~dkZ}`g0R)|Qgv+j@oG{mWYgsEdl1=2#6fakF23LKZNwS3eO%NuI5{wx&L~G2w#V z`i%W&cHVpR7_cu8vLAxo`*x18jOz_rKl{t|ji`5(aS(pV1M zX|M<7Stc0DyafOK_81foSAhRpr)M+2AYgME@%;eyCBE<5*5;O&e{$c+r$-*~YgF6V zkgpU-T_`LApkplpl3R;VU;)s2Q$(pmJ`zLDxsmUnn@ElWJcumsl!qz)R8B}K`h#G4 zkX(9_I?1|vOdxdR#zR`RDk`kuj&&tRV4&e;o475yLs`P)E^-V3ZutHA7$T@rcJ}1oV_6~qmUU(k{7)=P>Ia$8JggD&Psuk^ z YDY?D*py7)*q$mgL+Em!FHk=s*BDNp333*QkS`eu^wI$ps3WTUp!th#_C9@dc< z1=X+RFN$bu{-UW4###SWH?eTMOX>w&K6>B;(q;$GC_5JvC2P@`-W zgGimvnM7c%^A$w!1BBN(cOI8}IH)|G`p$}+>fQ28~V>eN*MRiz^<~^(s7-c85SVe1lzxA3pW9gGV$>ljW4^;`X2ftP6Y1h zimNbzj)8F*(ZAP6(x-7|q9nJaHc8^gHWYS`>K)*N!?s9Vn=d%$Hp!mAgcBDDy`|Cz zQ6brVp;bDY>dPZ>caU%TK5m54%)8FL4(7izLx=g`EZFsWTRa)K`BKM>-NUQinX7y7 zqdFs8rL}?l)RE;!is7!hWs6$pDJ;lX~BJX z5^>Mby;lY2nmH&F!2wK6rUaS#7b7wPK?#t9s9!*B&1+t`@jmMZYDovUEG+EQ;jZ!u zU$Rvx8t*Tj$TwZnD?2TTtjHBBy}zd8F|7bT+at7pU|3FZqv9W=3e)c8mimPWv<~KNCV+!o9ja zWWGZOrClc~(=LxzdLX<#aQy5)+{&?|C(Dg)()GIRu=*m4a@a&D77JpQ7`D%6Xk2;w zxnjms>AZEeNjECl=5`b3+z%l8(qXpqKvP`xjIEAv_w&LXrBy4h{8D!6d#A(1y;v}s zA8sq0eW07au`b4O$(Q}IbYz3mQw4f?b)(cji6qtiD$_zrIxy1C-i-IO)QDr_yIeN4 zcy~NCm9!4+^{_HZj1E`&o+;~<#83odW!0;Uj|{u!aW2g0{jhQ>(#5aEE+fk)nSWsC z(A8j~t2|rYakT`}A|VPZLxx?ayA_l+(gLrBkl^CIi{ahz-u4;x&cr>|fXN;Sqxu%p zH{+qserD{nMSQl(obJcBaP=yyZ`hJNKvw4Daag}_D;nC%&$OeOviv!(6QxiIo=SU@o|(+F(^S5}sdXS24yF)?c`IEId^ zKG=c#W+ztrB9v68@|wft)_@-St<+T#c}+Kdn#(YndKu}SDqYwu?3JrCwtOsydt%vU z##pR2hD#DD_}T7EyCo57>f{^F22_%Eah_m@cMNDC;Bd5I*(~N%vHEGurC8>k3ze@d zPu9z3Ed#za<`?8!PlL-RH0HOm%!acz*k~iy7+AW;iesQ9E)5B5%hGQ{7bWJ1T5>L`Ce$21&>mD<4o61B%c3mIi$mS{v=`+B)zBlIjbUYpd^Zq-*J2zq zT_Wx(0pMJ$IEUN*1hiG>%XYsAd~?gm*RLS^cA!>PK|9%`5i}#5en?dt*vDCtUlS}f zR19a#Tn1(}`pg_>5(4qd3TB+Pzta0>{nnxsR_eAmgQ+~BC4_2oFM%d#y|(fss9Bh0Pg3I1v#EEoUv=&xofr!=v|l_K)f7xbY^K9`&S38F8juRo&2S0A$6W>>qJ?ls_!`i ze=jnZ?7}fC#@)IMfcCmw>Sv0A&;F8m7R6LyVR(W^yagLI}VI8`z6uSk9w z=U(jXUy*#?yWSN5$%}x1Y70D;ciHI6&o(&< z0p$WO+lw0k&cVCUG0Efibq1V+M+goY$sfjfs28RKT9{%Bgy{#LqPSFmkuvTRKC*J7 zs;#0jp}%2q>I&4eTgon0xm0r$1GR|)o0|o}cp`!nnC86Cr)3R)QCDH@o@1XkzkL^T z9=}c#ZXxnJwvoX0_t?hxKVlo$-o8rh3y}XA+b{~n#KYJtN?17teWvS>G>*Zmo``jh zTE;njjv>-}yHF`cZGDs7nw>X^eXjz9HZB#8#N~9j13P4@J+!MV-+-T4W)u~`Xs0(? zT2}n{qU^ZAkNhi`r85k!gW{;NI)dMmfBa8~T{n9%5K1-+5`SSwZFc!6Dpl<<2v~=cqbCRfiNgg@lS@ z&{hAgP{;p^LLE#1AX;X3e^;n2{#Bvo`t~=48kkyM^WHEM`8}YMAW`=JG4_tZwe{<^ zZ!nY0I5W0qY&)5;ZQHhO+nKR#+qP}n95?S;d)>3|J^S3cUwYN}GHO(f>b?K|?P;w& zlL%tThMZstdCjUHFY68l&w$@0LTRe3+K%<;bDol`lHEL zBrEt10#^>*V(6pwmk*eCKszdlLw^=>z9s)MP&DY+f|LzE##$mR}(3jA|rdudSx3d zAir}hvQA-Fot5YMPwIUl;OWd%rj|tC)=yoI;7gD&GJ~X;@%Ns5`@^$l!qMRG$+b^!uVB1X8{0?UFxKDRw>G_E zF0H$`m*n=6#x#2%%70;F>wTQ1-PO2B#onAI#h5IAoi9>LiiFWFTTO~G1Z3wkr=&c( zj^8dh(1~o=hvDx1YVL%H9-5HAKl&ZZPj|eId-d8>#^8ElIE*=N}F zauOm1T6d6vhXVbKx?E9`!jq?Yjx%ll=#4INoL5KN@hrzB4>417wS#Ed=%~F)tVFN{25&mhM^4`M{?%-)=o1aNh z^f7P2fk`L}o)r?>AOmG{~VVDmy2l6TQ4dt*pX3T)Ffh=Y3i_ zzupe4k}a5kGlncJfV4xIQ~Zt9No76|reUx=JjqA1*7nO`Of$@5SgXBKmw6(EwmP2V zevZ2)(hm%-I14yY(Nh1>taxZJ^q8*I%HUwIUb#$0N5$>B)pd1yrHlGjkn^8eTJoRO zogX&av3YZ%zhPG78)hFiIYGZaZEf#FZ`rKQ$}9xGKkNGUoWo!Di2ODm@=y^lg=ucn z?QpRO#cMcSVk*Af@?u!(12v##TH^ob(uXKqSs6r&7tmOKnT$0&+Yu}> z6BsHJNLi-DhBk8a@F!tIVrDjU;h;16`-WC*X@`RU9OB#$t=Xb21>Vy0 zNgWBxlh3l}IU&pT5Q^?Gl7C^&)ruIpy?VHsVE|6H<*%>n17_9$;z=tsru+U)uTS6f zx}sgyDiv4&8+Hx4m%*ac>pIDcV(Mm?Q|Olru8wJ{)^}pYFVU)mvoEm@AD?x~$(>)h z9?oKm#HHWzdel#gzmrSB5XvZGicn00KAipm6@Xds8dzg&Su7mI}s_)n@#0Zloptm0OQSN|BTj zo%ERcm{C?@GgGC$okuj+3++Hl*|~1wCs~zZFZRe3Lz)`M6zXFr3XKl>ACTMV5L)`$ z26m`}R~qEh2~@}CLw%ZtH!)sshLy@bWBi#yY-K$3$Jo^T9D9-^%oe__Ay>Uglvv6% zrUZJ6EI!~~GnBU&6ZG;WXe90lKYl8c|1G9!{kW zTdm4rp;}3kUN>XV$g~Fcac=Z$VUi|vOugwjIqnuQQ-OAWHm}t(@+X@@{Oxp$dttzD z<+@6BYDuMO4$P3UFo{r*Xr5BS6Duw(UfFR)M{A}e+BGbyZuO{qbR6Cfzr@J=OtXEG z%}_o%=|ZWRr@!(>dyp-eWPpe|BbG8{~cYG z{tI1m{)4W%-=)9*XLMDtC=m#U#{Dj&$t4 z9tpDpc0~cRH@0;U&(8$d7qH(Qecq_3YR8aur~g6M)Ul7(i9it?bmAxM+Gr6-)I>do zb0Ie5TW#XnPDisX3`dCkEifY!njmQr>&Sm}XywD%>*sff{J4+i?dxLz&ILUDGh8;T zaL7AEK(A>7ujdlp)pN+_|KO-N{cj>PIbz4i^9?i#CtBa}a3^j6+9n3-y)L5dfTj2k z!aNIw>JW04?dw2?E^H`_t~%kEr0!KQAeaCo*s$ULHPpu-b6N zpey(Iy$STcWk#2^C!y;^l|)S(2xVq}_`fX1`XZwR zKXiae;0Qq?kyRkwUTQPniDq55dbTCXpu`U932@rGd>dY)w+#V%$e$d%$o=1Eineo zlE(N@NH1CIqpNm+s;iL-dk9Hj0j&P8es<}#rCR~(|NPXVBNN)IUEFmtzWD(3TAZ73s2k3r-xwir7YNf`Sah8vvuOljjVhK^ zb`+;kYwgs~s-tlmj46voiFeV%3D66>k91PgJ8ToDlVdTpv@C;VeoD8Lp!W1s)v$T{ zm3X_P(j;<&LmX|-qVasSZkXiz znJp<0v$P_d*`N%z3c2U(^cQwR-{J~kNlfA5Q=R3;d_%i@L(8dbx&J_sUHyzCVrZ*c zxl+f3sC&iTXGNWNh2*Qc5lNcJM@7qF>AiW#yK?QQ%Ffj4Xhi+IM|~G5j>|wI4V^P| zeVyy3sg{e%Oo0%eBs2;VaW~IvA~x!VU4Hpi5Q5a=DO!^l)g_9H7`^FrB}D)0tbQ@V z``Pj&(?D8nb!yJqU$*et;!fgkdE_%;^mk~u6f2N2WZRi1!3OxZE@$rMV`=?;+5|f6 zM{PurKIafo`z_vFkR55z^!`DBOxAvl@-Ci%`yT{OW6?dF&IB7XHy0#i;_d<7Iy^&i ze&^lp>`E*r13T5Xtm%TgOi>BEZK3Tg@#iZ*`Rn@k;LSJv=Egn zg=!Y5EF>Gy?-TNvitGM z+hMDNkc^jYWhk4@W2=sMQzoMnkA9MK2D5s&o-3@)wkN;o(t6Au%1jevpvIseLW&KBRl!Jv9|Eg z0eG<9OfPnBA1~yXB7J+F_!R3kRlY@uDX7!AYrYpFsBehXJ$wvURW$jVB=cyXwv@2oKf@}A6xTztfjFIG=8hG(byK~Xhwz1o@d=atKodKxPIg5=e)uOEo7FWza#5s3Cp!0H- z={R`v(Q<+UMX%F1Nj*>Lk|JcNO8Mq%K3hDmOKbdV5T0}A&=q;={z zdFGy4Zj(rk>;W0p*i>Gj;-ni9;LI6_{uhWgJ`V9vO!dKo%5+v4!%NkOz^=LwrIGV9 z;44j6r`Sm{I1b@Z2dMsW_tLNxbj3w{^&!{k*=c2_rEL&b6JaIca%R$kM&!6pg_z(& zmG0%j(AdRRBA^E_vE!aBd{aQJAs`2lDgJ-#!AF@z>}#8S{p`WV1|jiSo#|z1u>O7= zqV}V|bWIg?Vr=$gQRSz6L-GTMN2X<$`e~o_=O!okn9B@-fkl81Yo8eWeP^$uuO|7 zrb;8OEfo`EHn?Vq{f2`-x(Kkzd|Y;W>iy^crqqMMG!oRk1dkF%lc+G9o#DDll|_#b z{=25iM5t~sONR<;Fgqi=ogqYYTSZ3#YcQuOX()-wf$O3_H?r}jTY;(0Z?HH6X0TX7 zz@i+5dG>BS)ubHtk8L?Xr)uN+3O6L`;oi*QR{PSLv8qB zX_OLG9&=2k!IUHK_E$PiE9Q?A+SeQ~%o7 zMFcEpHP}ZlRuc{AG#(p+osr+}FyPU~-n`|iY}UJl+6KFHsBjpOL7?c+A+@%qeV6Z_ z=tv>-c2pz`Bycz}?{@%tWjN~GTKg0KTKfeAEJ~4oe;eg6|Ap}@aHmmZ-D5kjm6$-pDFtByff%YJb1MzcC7hruBaSf2NLqz+b&$ z0EL_xRx!2fM_VG%4QM_tq64yG+om|T7QT%^KmuwIlt{6wuLh|spRKylK=?61`O=2b zH~3$cy^n|^dVh0&{99?269b|0(dWOc*F*nUubuwOdfnN-^r^C-f#T9?WK^Ie($vU4 z#Es`)@!y#Lf45f4ivE9Es~vuvX}3Eyk1xs4CQ~WFW~kET_ig`G9J`BNTr=bJb4-;q z3pDL-fkTtck^Uj`J!OQX+AK{utZRp}Fk)BdPl<8>HW3H;!dNbJlpPprP(lhSJyR3e zmufVD{Fm7za7YDi8D6yfy?g8~z%R#1MH<3C+6r?-uU%znAYPnW0V+W|x?+(y_uw=f z`hYq8cilsCxfb@f?LkB$^`pl%^m%RLq9{3SPEVClM0MzRr3&F8_xE+R0u z?M~eX&A5@1)@1joKV1^l9kE?uL*uc$;+raMwayAFYU@msWFkXH7K4lL$2^Wjy?f-v zA~}rD8Gmj**M+y>6&DM>kfe~Kqk$Hz_0_Oj$2v>XbW}IK*L)Jj#*%!Z%_YYxTlO;3 zx&>UW3Nt`pj&81nZx;1JD1A$*_V7n2huYTKYv=dhkCEmE1Y@vHzQh+rGE!Vd0?VyH zS(345W`c#$IrriKRB@*TP4=jz>P&G5R&_H!Q!RCJv;=n-L*X3z0reZ8R0>wEh?$6~ z?{&Lko5Z$~_jaL%Dk?-~%e6m1M%OA&x?to8WRuap2ggr~r1`&Q-`&bQ=1e07?VH-fm`3~^w0WlF-sTY1p2vDpUC zclK^=8FFm+uyeS&x$3Te>-}K?(RaC>t8HCZf;_;}XCMIse~FfE$$I@6B#k>qAR%tG z6_2$+zTBRn+^wnS`A?qTKLch(zt*pxBLGj|XQaRXk0TPCa7chj&NfTF5q}Su;q)8j zu~NlGaGKcPe_QTGdLc}|cm;lTC4aQ8Fop4oMYiSNWjc7m0ltwTMS7dCqaPQZQd z5(dc>y{Kr@x0Sl(i(?DK9w6sFzhIK`(9?=BPFJ%a`50lP3F_e*v#b zij&8w*2@W|f-%sOG9SLqXY8vQ8(`MxZh+bn>Jqf={;(Vk*!A{N#Rl{T#9;tje(}4V zc(p%*e(wnO!=@XgIrB%N2%7sVz#ssW5Pi*$^$FQQ%$K$v-?DLED0{O`d=h8{gUiB^U?#_rk5zdi1>i3ic#A(A(Lb$>o`g^JOJ&%ibQ z$KOsGZ{2F)=UezgIa-4f+2xfJ`#B5YnvjG`bO)xk>O;2_p5?c~cY^@()OhINtmumL z!?9#jVupm%Q+)aM=73hd!TTT59ot|I?A1iJj{GUvuHkyWMP;3w?G1bXcx55WQ2 z932^yN>L8%uxL-0c0#cVeN<7;GslA}{*T`mR3`?}EA09>u5h)`q$`7a6x+^jZDKv5 zWT8q-4ijKm^e0Nc>{5=p3=f3YgTOHPaTfGOiY9sun?%Ph+3i7^@p&R}L(<@B5TOU+ zB$yt&q%1d<(q%Md!A(T(3GBWX4Y}-@YilRP6HBR%4hZEvc#u&?wpo+unMWqYlT4z^ zjTp;trPNg>y~l@X5Qk$I=9Nfh%kwVSB~p8a5z^4voq%1_)I{;VEQIZAP}Sl&nsNXH zhT}MUXS{IEree&GZ=w`aoJw+xzrG_AgRg)b-m{EX0FA%#XZF(@$d3_EaRAq2+-7BB z%-rB7c^wAgTap)5u#v=t*-jhrt;vLUh&JikDxA`+BdT4}__J_@0;$X>r*=`ZS`DR@HkWAJES?ypz`RabkEbhVw5jaR09D7AZ`o$GaIaRNn@7oL0$kBLqq;rGqEx zBBZHmU3)tI0dSq(6kn)<0I?Vrs!~9{o0rs^kB#gCAmD4}1knDoxd5@ zr1=Z>$RvUl&19ApIzqhDiU5%Jh}=5)Jo3x{u7y7sR&fMLIi5yNF<^J{`JM@DLmUBO z5+@G4A~%c=GuHHo*xlw67`|{LEXEwd?JP#2_GTj7c%oia>Fm>@-XkpT*7NykM=6!w zUX|sV9iKJ)?YuH>hNQ($NDKGQ?+nxPqbEk1fhF6}D3cF!G7NT!a!kJ#<T*VOp1`oVL+>~_96%ZGb5i)4-0|{ z&znd)%k=mh?CvMS9OzC%}14@|Ko5eR(Hmz{Y11k!r38n_zFE1?O>W;m7V8J z#xTksW#70Esg%0`=`Z#`1cZzH^l@1@jc{ykt!P^WS~WehXty%Ax!^&Zh)GyUSt~J2 z?<(T#$MN9_<1H2OvM%MP@K(y?6o5O~;5RvU^(hrFy%tHg=s&69j@pFQ(h7{92>MES z^|{np@|plOtsJ-$S8q^yJ!;JiKoIMW6a)>?Rj;iLZ2NCkwvdj$Zlh#UHS zW`z20~m`IHZ#* zA$nHH>)y-QGlUVjz+Rs{cD+*R_B;#p$s}Tlt1uB&ZJ(9uYVT?LN$PdptO@+R&-x%$ zk(hf)KaCJ2CjF3)>=nHbaJVlM&r&5M+G6{hWf_FMzD)rGLXMko^l(i+{?o0uYn{c3 zP~&sGW{D;jK$>l@P;+uu6*9Zhz2dAA%4gKRDp6c6H+7Wjl`l$b3G=u{OS!k7EQx10Z~8lX^o`$Z zz(+mu4&%^FPF}_UEuR9syITN^*w~nWY);6mneG}y!s$nwU@OEL?tFq26ot}+x|c{rbWPs zJYCDd57}Y>+Otpa(N_p3)M2XfQ;U4drffF}L`60weJrl;aJBC=-LLtLKjSv8B1KI- z&IZ?0G%IoJ4Ns_{G~y#DJ;7*Q`X3+9Oty_Fj~W>+r;ztfMpe{GZrf9Shy@qD@zynM4t?^zzk3LM^0J|) zKLWA>3qQN{{X)e!MxWLk@WNk)4Tt zvu*m!iWdXx6WT%e5(Xj01f_8563v*IAPM`b@Z^o_^qCM&Km`;$F^muOtE9A5k4zL$ zF#oceI{hU-zem)b8NPlg{mIFC5ZblCVwDLR9bZai8PNe;zS3se}0Q>=!k~UV#27Go9H!e!$nc2 zhyBpMm?|$w6J7K*H{C1Hi1@9O{uzNz2BS_@A{@!cg`>iPqAZ8;O#V^~Nvs_7d*gCXrR(-ZzK%~LuP3q68F+~eZYn0)%$PK`m~5v6zxx|U%8 z&jIv1Ez5pXPQS(%n{HRY@>Z`OJHUbT`&XmE3l8uIgUEHOl@M6}L`TC=3=lqz(d;V^ z?M4=0mK0hVFSJIot%dx+#*KukL}aO4A^Kru(fxzw-s=x-C5YXDW%dRL8ZFuljVnzC zDLzPht)&pF@uA%B*c?-(mUJo(aoTzHw$?rjqyWxVmGYvFDlM}X3M!D}cSfueQ;nu6wK zL5R$aNU+3rf3(2HYLJdNS$Tmj-2B#igVU(?Rw#m8mwRf+zcj9n zNKwphwr!a(QEqj7wmSgMq`8w(G!mPDME4llE#Sqg8neGXqhHdmmvZpOdqog-(v>w) za@()Bk6h5Ku*}`~Ws^7pjT3Xw0jSl`=Gp}$XkbAR(XnirRzhlyJ?qG~P7E*8#Z&gr zwnm7?ms?WjaZ+0kU6tAPBb6OFw!xvUGFcN$Mcpx!9>fJi3RX!{Igc=7$-K9PTI`hK znVzrZ?crGPG6#;L`>#I1qehNaM(YCj%=g_L7qzZ{`K`ZcfNMh2OUE;g+2@&M^mB%N zvC%+JDTeq+k!zirKD934Ph@lXemOpI+rJDF zqqI22^(OuXOnGRsF(K@LFyxtfT~PJO&2WZPkY;Bq^=o~0pBo(0YEVQt2>yUrrcVwF zH&oz{tu$vjffu2O!Adfw3Cu1CeJ2`&hvXa^?Z-U}xAp#%uv9LnzfEL-hho;z1JV9? z)6|8J(lQr7bL?@T$@%$Wv%@Zci((3c6B2YgJMBw^`t^c{Pt)^sUqfaPTog*sbWd1@ z1SIdfFCN*C-8Uq!Hv^D}4ktK47UeHX0b_^`4YuT!VS+4DKsMjdB%L{Bk|E(V{5CvS zUr@fX+Rn1uzn1sjB4p-L@_Yp#K8&@;190Kryx?^~to){D%l$yPT-lFa;2+FqvVTwJ z;`&H&s=d_`#2bIs0qk#57B(zG_EN{PsC@36o* z`kVZRn6&TIh2k#$&v9t?w5qXDNkp_qfX;*AzQc*Q6WzsgueQ%maA}-FZ6Bbgw?@NQ zO+T;dNOSu`fhl0we)&NmS_qQ_zTvQS0!Tpd{U2!#8$;%72p%+;KeYOJ4g5IpX-)US zMn9=|Q(IP=>Pk_^D#x%i;yk!ZLkddYN_-ZsjW{5Ha3zNnE}#a65?9DLEAF%BThNYU z{ot#4KiR%+ZZDs&zu;ZLsLp9=8*a6}ulBFIvj8_2A@09(OH9%*$`@Wjx zSjXqtL`Q)9i=%(OnkUU`kmb`H;^O=wruxGZ_3L?9Wg5?`uzppAgA?HHN3u0_pOqO3 z>`UV{v3H9Wl;6R%VF*1u1J)1X2WWPAVrUhQe*R9O8unYebZ-K89)r^9{rqcGhC=&l zlLvX&g(eY0$D{>#UtsUe1oJL&=z`8vi}2t(yYTr8$osTmqI(8xN~x{^1V|AuzP%Az zw%cim7!N#DNB61Kp6a*_3s?`1T1R?!Z`k^*n-}9liR^f+`^fw09PNLTu{u72diY1} z?JCELc9UswrZFxf-7I^#o&+!SWT+cb0YvdnEIMJ4H`v#sO43i zl{Hs;_u--jS@c3~FS2XKX9P#(1OUxOG0N*xeB8#5h`8 zgi_*?9oK0U)&>I`tKE#32{EKQ(*LN6XLDrZXAs0aSKG<05=vk|j%9)3288W=Ju&ZRR3r=@vE;6e|3dUz`yRe zr^f8nRk4~I%8VpZtW5{urq(zNeMXcaM&KRQ zDxpU&*~gaU1{+LEgiy;-#alFAy$#uk<2Zz75_Jey0=%XCppy-}gdwF~De7M7o@&?b zr<6MpR_`j0K!z2mY19N;7c{zbWN3$_sp!~e-_`s&5SpPz3FN=F9#rh@hqEB}32pkeaz=#_rIfi$O-YMw zCmJJ81$>A8<@bma=pY~Lu03ZYZt^9U5v`u~iVZyKER=6|$<%%ySZ{W>%3T>RDSJD` zT9ik|QfVYxzbBh2fAmH9E{Hv2+w-u{kn|dner$!gz%mujV8VY`3R_WQw>F?uFTK$1 zjF^?IA%DNa?}Nn>#?wB4JUMXL}6PYVT} zzD--!{dFethHYI_KT@(nwK;J~s(CvsnBND&lC$h-W(1RVL8R6yW%?5f_m-w$Rt<#{ z^0R)|OFt;M88Fj2juls(NOA6X{1ECvn0_+EPVsOo<=Z;hBi2IAIzWuV`k`je zuWGera6N1o1LbL)1WMBG3`Fq>!<$CRK8W59CTo+U5PtIx`XEan2scs(87h3espT_3 zUJ0tReUx9x)|1$!fbZeW`_Mtu%8mh|!USmbBdV`7`qh{2s+^cY%W}J?a@g|uo?#z9 z5HJEaIfF*u=mCeSUQmUoN%22Xa}I^Y>@0|oW);~OsKWWz)WOP9Jctg9y&JG<=tCAw zxHx$lFi}W9H>(6Ocu=6122-Y&pPbzzmQt_@@tQj<;=k4rSsLOcnVK3 zMLt9h+`~ajs-R~)%oz^|NAe}y+VB>us~Q#$6!nZflMjHZQgBe!oO#c)hRs8jugV53 zCEhFK+>)(ecyIeAmVM~G!ze6Lt4Y!#V8#pNRlXj@3WDFyaAXO?P<~K4iD^JqPz=u{ znK;T`9ky)#07se;k-e3k%4Z9YS&>&G?HXC7eHu9Z1EUP$HYKxtL`^Wrb zS!I-Ao#!DLb>3U_!Jh4-B~NfOtg={KNIS4trJq--uEdcW&$pjUr8UaXPH6uvktkL) z{+JPIwrNKJIP{P%>`tm0N3mM%qD;2INhUCafQcyo_W3%JBXY$@Sw>0NV=wDTo6u)Mo$SX;yU;Z|6rIsE~pR2NwpSoRSy0SMGKMYgm z!%wFj?}5_-r`!{O9hY}3sT{dN4dzn^gqVA=2`JX5DdR)&46}Msn35u! zqdKgox>!N7rp3HYJya|yM*X2YO2s?)Pq~qJW|eiBjUf$<@Rl0TVG4u_&5fDSw${n_t zc5xX#^ZEMtN=~OM(TkcY;^%Mh_E8kD;T@b+u;aRl;hp7%(C$ss*TNr#T*0b6!zsKJ($%WbCBPJbrNqZxZEX#5~RD*jSgAvwAwb0KY9H*`acr zuRp3r-&rPEOUo2!rEHP}7Tu-?dk51Ar8f2Vw??txa>H&szs8u8(Jb|`XY$mm9Onc9 zJQ~Qk8C=9$II$Yl;4g~`feFM0M+md5cM)P0!QXG{0(C5vu$^!Undb%EBF)lv)5{h7 zMv5%G>UCPvt~2&aln=i(75^znS`hmEmX(UCPVd$gtR&L1;G;mM!c(PPS0;G6|nui4^PuPX!yJbCb2IkR=s2?p|*u@jiotfi+N&Z)bW#t;{1Y=C&l^}SB@5+uYZQbF2!Vl_iwk}=~O^rJ& zJ1YZ);6M$ja{_`D#UUn<;NGJ_R-+^-9C-=Ms}eL3%hgr*_y@qyB4Y5qMw(q?u8USI z+ecmZGLBE&(ba0xLre=PLL?He=*ZVNS}kb})DcQWiQPm8*>q>`WNns7$Hhs5*VM(R zUf|`(uMI1{>G0IhZr>^WsHzGrDy<`d@i`RLDtQj}ZV72Xx90BLOR~{8qh_n=?!;?? zr(=0VELgZvc=@Q|p})i#%>1A~8hJ9gTU1swaAo) z?Q~2Q2@Kl#U$`WAzZ|QiGd*IBvbtA7|`Cyf*KLr$^Ky;z^T(k5nohT zIZo2tk*+|KCsKlSH(g+=^hvjAfhK@U+r#kdvd1xb9$VOp8sPO@0#IBqw zi>bJ&_Sqx?oneH!)49Nn`)L8aind#U0MBdUaIJFkBKb<##uLE$%sC6j7ui;__ z+6M;i4Px;5d(3h4bErSGmq*$r~yv)JNyB-Eo1}Af)8$yV#L-9&E~$!>19$Y+=~a@ zpZ`ks;I#Fe;>BUm&*x*i-SKNqhrj@z+vv{VT)6zxx=J}ZIhpKvbqFC0nvEBrqfsm~FwiG-p>W}g>QDCY0^l;ih+>a+hd_n(2t-_WmVQqgSGDN+qUf8V*{Q(Hc)9afk1tGA5M z9Td`dnGDi9Ott~qWGvElJa|ubuf=zus;3A>D4L2-^EK3^{x6HU%dL<->v9AexLa2X zM=w>_4F+s6(g893*e1p6=?t_zaN7M_e>;;L8L1{d;n!~m&wc)j&JqSU6d}@c0!6DW z3Mnv&w&sZQnMe>BNkuKBC`aQyaDi`ZmPEUM`TVsVE)00W0`|w zC}zBNK zBhqo=athXp_E8K@_ymJk-0J(CLwdHA@A2F=i0;@T|0&JA;ZF_#{?($0_~7%3(8ss` zqR<6~B}Jwc!#A@+d(x8rfOkR76pLPGC{}amWmvW@H}&{?+d7TLR-)DXx+sftmI`Wm z6`)|Z)E(&4>&1HeRLe6BbsdPCWwHEVQ&eo3II)8FaSY`IXV7}7h?pohuthi~i z*fO#Cu26RF!OKkAUUkL&2-R0Kp0ByqG~JL)Qsq{HLS}@wdXQK`d36qxeNEsTcmJzZQTrI}oTIgEQ>;_mXlXcy%BaDSra?%EvIuPs>^q!x+$KyV zmEptIX%=#{BIX&lTPhbs4W%+!+0a4{XNS;?@w8ZqbJ&W|+U3iBpi9AtpDAs0EJY5w z{Jq(>>+5#o8;TM9J)sl0psqX_eMVkoEZm@|U*AgNh@2EKTd*-kFn##$I6 zFQ$BGXvSoN$~xI3#+*zgS~%`Go_583M-qDLGEhX!z}%E^!vr(Y;c|YjZ7r>YOhdhWA}~Ss>(%q7EoWl5l@2jqxjI7I8tG%+RG|mK^&{-h`3y%(9$a{~XOk zZkc1<-mb6QV{jF83Df(~c77!8=z=aqvep{ zxNJPxmFd=cXIJ!)VEndqpr}j!+%=geo%5`sVHtJvYVW}r+<2{*i&UpVv-xbviQ{cu zcU%OTSJDLLkyw@9g7^~>x6nY`ttZCBUnv`9I;g9x~ zu`F^?Id)9#m@ZX`FQvg1@w#>q(B0b)Nn_m%f)Vo3snfz_~E@Q%5%I&)1HA@YMQyAq*g?id}`$%w|}djDK% z$K?HG=jg}7+Qi;uut)90I>5y*nE4AR3H~I$fZ_GD%jJIvs5fT;xT<2lvQlL?&28x>P zG9aNyfw4F+d0 z>Cz?fdtzAW&7ht2QQ8(I1+9(o%d=gD%*E!4Qo?)Wa9NU_&Vao7BwkmSOM??}P`*!3 ze8ixA){k3$W&Z(49s^bf?iDiuq@2U;ai63#pBl2|Q@{<`?7qkVub->%7piL+7k4va zHcw2aQqFmYH`7^nM^fP#Hnl;LqCN$QGnl!qC_&H54YaC&akb}FsAHy^fKmP`rhEh; zRM*s>D`X6d2@ppNFh5e$d4XD0J2ZonYfCM_X`vFhHMEGK^ThLQ+@}$tPAtIjb3$KP zt_|y!BY%SVxsgcvsOW0^kr<8Uw1o=(Ne5>|Y}c1)Nz!%CEl~zFlLh7A&ECu(SC+$p zN6o=(NH%&0InBTNGlU(2WCjFb0MgReT_Jij=ei*C=V||eUXoFot$xJgwet14+N%}C zk{y!5hGIf}1c5#nLt(D4DcEm|I)P@Zq7xITX^!nndzb1Ds$S8|#cvLwen2iri+aiW#O(@Ci#&Rfb|VP~wsI9;D+ zcUof6E>o{xN4v2ISP%P~68sMD4RTg^V$OJt74&Y7;!FlDXGQ4!0-=~Nts6h3p-HA? zesey)?-WdbnKsXx{N-CUwy5>WP@|S=FuNzdI8Zik(L|#sa_MoOboP<3Q^~`+)+9*; zK_W_lK+=Ing@Q77U18zc87L6E^MVXbjNJhT`)sS zp`p5`AvaIYMlLC;)|VB>#$%PpL{lUvaoCwWc0Ecf+ID9Vws&Ji?bA%{6TmeCR!Zj2 zg;XlILId2C*`jXG=FR!rzOZW?p;KfbmCv}u1xm=abPkh2wxX6rT#%%#`fyKsd@%dg z^-jACzdLFnvs6j|}qXVFVl@ zR+_nJ0JrB)bli4+D=D*jWcob?q*}Pto!cKjtU+}JqBR}Xe#!UtMZWzlvtom-e?l52 zqOoUBh_+;jQ+p2isr!seA3YpPRGzG;aw)9tl)lzbFkB_}%kboc>@Zf0jW1p`$DF}G z9VR$dWAEJ*n&F*xCzc?ER$b)%xwqAUIkxs?S&q&d&P-fI7qoxIWVSxy7Z7In60i`3 z$P6#n3y~STU@Lu7Q6%^oG?G z{lJ5)0{G2{1JEYqTpi?iQg<8hz!@#a#FEbyFd_6Y@X*0`-@(w13t|*9%PTw`T!RQv zt@omC>&4~>y|#tkWGv6pZTiod&Lr*}*{OS)se9^qa^ohX4TO-O$=7Q3w2MVD5y&@E zsf0t(p8avw~_au%fTAG3PKTK3r?0K?23?Hk-&bE zQ`jQk?*ZqjiQTz{UD* z=RxBm>t~2#$C?eI$>2@O^*TWqLPN~;TH&lIabLnSGeEe<{YfF!p!*zTrmCXc082EL zI8afR{}?UUhpZo%R(Yb*6O|uLU@>VKY*0|-!g7@&L>Dk*6QPqG`H zfEUN$=uB)XY#|qbG$kW@#aF^~0nct{@IE#Y^M#=pDX#$?5t|A@Kmh~7hMB~Mvo={? z524exre!W>9% zo+QL~d^s!v92V_E)DQGbs6ddD(^gg*sVKyANWEao5B*K68z;)rk%>n<+<&7>KICC* zv&Ge2qJVJ^nomUfe#{}I;>uupoKp|tUN{zL0dBD~nQ}d2&JouV@#AE2BAEl0tl+$n zzG3Oa|J=e@gBQ;+ajmB#+JDZJE3Atyt77aXCzo?(0Oz9^$6(&P|FXLM%<$Z8OrTGl z%OTZ_9k>rL^VSHq3HueReOUMU_{~`wGQ?Hr7>m(X>)Tck#ok_tp_j_=xIni>d&SH-r$foOZy-E9{1AvpF>2gTdLTreh1wE8aqZVuuMGxJ06 zN1$VP{vyow%!|GSNX_2c>wQyKMb^hJUtwAkhKy#?UML-w@X>4*VyIW~3=9F2Pl1ohYdvMsJpSnsLoM>Bi6P)?O->$^Y$T5_s zkTb4>b^7oLf0N9=;`}aUScZf6tK~Kv%}r}e_acIPM;nCOMerAP#{WgBJ9Ld zE!jii2#nx2{e@YH1-`E4C#&B{N2|Fyg^1`#Qv*ZH%~PZhEg>>vLO>ynV1Qk8Ko5*o zFYX1|3M3>5d^BBP zvy>4`IHf}uc50-KK08S%D&G1!@qbp{TfxwY{PRkE-si}xhcj972lL%) zzHI+!8v$-_x)-z^P|`)ONUSsR9A4q#k76~QK@F}EG;RUU^N?p>Pv850CpS04_5j-> zs!ae*rg8n{$8P8N?m;|{6TI8G8zML4pZtF(Zt|Bnc>NL&=zlKhcQ?G;v%eYlJ-;aj zQ`@7d1!3}bTCaz>(~|e}c9v&Hxs6ympDXl0Iz=01JO&)Qte*w9@}#e3JV!19E2jT$ zpEvS3^ClBuJ5zd%O7yJ$bf=?b&)gB^>$S&+2qwt?B=AyP!(0t8zrS~(hrWr+w++Uy zYw?qtwy)CRaNyjNrfXHebKT!?aNhhKBybG`CFJD3j6Nl0b_$oI6U``4Jp5@yiPDA^ zP&P%e)}M5iP|Kvlxt~~-v+p$N`z1BEm4mDasT57W7QX5I46R>_++oMBDL6VGeI*FV z9jXA2nuVGI3!Q5W$jvo0zwZB@JP_Ls$o4#K0IdA?Ol*o3as9U0-u+X1UL_h#P;GmU zmlh%RfU!NL^=gb)(J}SLe50!bn7XYNq$RqpQa4kMD&useu>W>@gew#KeOMvZY5>Y> z&V@GQz684gF3w(F{$K+5$(5L4|JVciQLs1f4tcgkOvUyv1Vs$&?7AJo5e zF4uJe3@#DhtYmoth+O`u9Vsfe=fC^EXxDzv6BML#0-0X`Fu4JMgOd9HZ$OjyR^S5o zz-wHfcEkmnL~tUEcn)BMc`&5MzDRhYDJxPTTwpI!qQeS$ZQ#O#T~$^h`91$_;ObRC zNb(|ts2;g)%GNsF7vSNT%;)>4ss6Fw?l{vBfOy(5jdVN3y*3ue0nz3F_bJEnqt2^q?Pooh)gQL z{@4gGoP7f1=ji7cnjZzsLA-zdlgQ(K0yxIE_SHXs*}%<(8ONKz=hTF8uDPR_34X~+d1w+SnKK%%>P*BFVC{i6J(vFW`SQ4g z=)`az&ljxVeGJyqJn{Bf7*p>N6JNDvsVkVg=kKawP2N^MNDlVAb$#J;s#aQ-)wuL- z;wIzjO3dV3uz!*q#Ir85 z_MNv`?IGQ2(fUnL`*byovnr$bm_l%UX_Yv}I=x=JnkzEdI$h)%Hx8K|4M4|}AkY(o z9riCS=lS+9J&E`#c5jxM6FHm>HO~P{yA)!`VzHk^-gs#<6`Cyx_ItO~oBgWWw zy99l(wH?wO(KY1BjMi2AnmGQ+ZWV+$G*s7Ov4(7k_h#IxZ4v3kKqbGCHepkRklCqk z&2^;vE9Ne+=3cF4{PD+?X6jKR`8dAARGwloD-XMvdX~C2vGUScdhdJ+ht4<-BH#NQ zMr)#Zaz&V7zalJEY^uRxmbEmxLMx87YCZ!%&-T2j|__jPfc}b$cn@x7`0*6)ypa~ZLF`?F zH(jTmjms|Jy~R&U+RS2K%SgOMzr_<s8(^eRy2vgd%9QbNIde+&61+i zsU(Ux+2E-#o*55g|7u+iyXK}nu3NBvJ+6O`Kj+W?4)#gLodcrnqcWX86AbW@Xf6xL zBolw5s(1~ZHnC92^7L-wa*}H9AYxO!AoSHbs2JjSutaaPtb;b&NJ(ztsy>z4=sb!( z>BPwz9fUo@-LxQXQ@bcOp9Kg=H&~zY;oY}-=4G1S-yBkdzG!>Wr}vj=<1J|YWf$}z zh=0`~(H3j&@EBa~-$pjz%?~ZmcIg-VEo-Aco%?E!u^-ObH0mM)ZrhghC@l$#Pentk z?gV1FYT<5OGmXz*-K+$3+NCj^WfHNDd(G?8c(sR(%Kz6U|7n*Gw~*5R8nqvn`OAlS zu{ze#knCGmj{S*A^od}Rp(kQPW3YPpu;cFH^|txDV1IrGuqW8dw|Kvw?KySWZ!z0A z2T}ZMd>x0EA-ebFuGVyb!1>RDgoc@ECrts@i(9a<2g9E4+SZcOxesPUy#CLVY^fip zbRGf=`rE7}+^37wfN(nw(J8rdupul-(U|CzaDGQqMZ04Sdsx^=n}#VGGI?9>cbUN9 z39K+T0PDCl&m(E0>9*Cr)U~QJFRMX+W1`+*WkGGB+TmC)lVfu!9fh-rT?TWO9aEe) z60;uA?(gOCasMb7@QD@iWmc}aMpful-l6oUpNrZXOlb|90nu(EXpxbOS^`gZy} z&M6q+|8c}9=>O@2aVwyd!=GQ2smXdPGDFK%1)3qvH#lre8UN60%E3XGKjmG_s!DOM zU-!wS9JQ9d51A~uMV?*m9ER1@8(c``Kvo!eg?46hQh39&0V8U$vYUEuq=9XK*>0n+ z-Rz_9ehcJ-j4kjq%R6E-WjkCqJ5gmo-_B0TI5E*|`j4fDtN+uWhn;>nd+t3-ei8V* z?5J4vr%t`pmURFcc!mP8l8_aeWUC-e;Ngi(t8Ajy-&yBI1Mo0v#1W;T4bNU&CSH|y zvlEKWkk7uDW*D$K;%4i*i&J7{8{$A~u5EwC=K1^4psC1fR>Jm#v?Gi45qT7Uv+}ZE z*Er?qZ0JoBv9PoLCTS9y|H8s>?)&Sy*$=x)qX(8u@`6uL#YfC`L0SC5N>J9OGxn7+ zkcvEoe;RREN3Cq&L~DAR%An`F+imZ%wGZl<)a20D2o7j`-n|yEdZT2y&0A_{6;~C{ z%Eo!EIVnF=gV`biYQu8qPU{*g@=O9dIE&um2E+x~KYEb;eYqWeQRXsyvxlSu-8nfg zAEYa7DT@VNt1w*1DIK$?vM$6IL0dBT(&!#+3Z}YjTJJi19doBXa$bN$@4wyyP5Rn| zT6JRQNXCkqerLsTQJl=`YARz~%W_3LKc&>3s3-%Td64H)ZhWO1{K@XA4Qn(zQ;k5Y zzJ0hVjNe*ve}+=ZsFbJ>8mxu~!ZHpWlxyPOJFV|QeMKyoZ|%SK*(3SsTlS{?_(*X7 z*m-(3e0)afH-4DMH+=@cdS5{|SAWY*TzwN{n46n}$owE=oM(LcO2GBXvpxYx%{R_{ z6}kjmIo>1?f|!GM=l&z@Epweq=$(F3&Nw9HzjS7(x-HflT=V^@YRJvLXZ1+snIR0H z+o}iHd!_Ix2#U@Xa=+Oo-&;II&dd`O`ih-{mZUMU)(CD@Be?9JPgK-?t|H2dlNk&a z1i6h5-!5lhfZd`a{0)keVOXj+QgUYlGOO&pi`^#`f8+pBO552`L}_rr{g9?-xKc~b zWfp&`5riUFncIVvnXH9uu%@lB!hU%h4gc0;hLJD>xW&RP!mF{UpjpzB-p_qo{d6Qt zR%17>z~P5C2BsAbGilVBo-@UQ;2!}EGbv;fggaFYGa*O}E*N7qKgGCQkN&^svFD07 zrf}x4^e%3dfU&?h*9!P=GCxE9tn=Z2>&)?2X8UiQUSA}FR3G(*dTjh~M+WnOytgPMWlcDx~;}W|=A$ga|w4WoqkadPN z^9{s^vNeh?dNrIXf{f^wtf_AFYiIWEnxkE7Ynt97mjxrerOUg z&z;%3%8mX+7RDMBYZe5t+*F-;xN&}{SMdDKN)($mIf4sCemCYGYAu9Qz@e*??$<_) zf@TdKzy0d*u9}DYTJ`d~H}z95w+8!`i0(pAE;k1^I2-q7fVR!|m?}rTtrA>2uD3?t zYc{t3@0T58ER6!Z6UzVpzcX{^z|N)h18n|xm6*E(J5Y5CxG;o!FW>y=XHZml`ZCP+ zyx0Y#X8!U6&gcWOjbH@&;67f%?+2!9u6jZ@if{tgGT%BeMcVq|4`;f+F-$kbsHOVjaeJ za2B}>TNiK6G1qvY?q4|59>3;_>`|+(EwyNlhdf>1V5igL9cAYgg&!N%vp@gvBG@eb zr*{ATN0UHg_&EES@ci@FKCbu0CgA5luvj1WjnDD!U^nExvxiwNh5Hs^d>qrMpz|W! z9wx*?W4kvb$I_!WU+=K@B}H8*__~I5Pw0CQJiqurD4bpRvyE^AZb3fLEa=cH05XRt zGJ#Uvy10A_oWmJhYFhO4=bX1aML!eE9jy+iBUr0wYB?&ru_HEDxpJa*5Kut!^m|08 zfB`Z$I#<;sf}#ZD>f=_s5JQ)H|Pd%sCc;d{R2w1qKL=h2QuhL%bR8?P& zRN~gaAPCW5ery-A5wnDCQOMC?D=VAlaym{dqc3wty+I(prKEE_rW836Ikfpj*3W6` zpcBMdpl%*v25+0)aYT!hvz!7Ne1kmds}s_6w~AqziRNM};scQc)BufKL=vgH*|SQ( zu4SiY3B$nP{YAcb*mA43!iJ~=H|K%Fz`y~I`KOXB4ikrw z&A_r}qt44%nmVA1%^*jPh9(P^fucJW1Jp*k#WTPNE05T5naTfOh2L5m6%g^!6w@vMr z&xlu<@c`R{%4h&p_R($>Sd_<|8`uPaSYXPC=SXwS#>(C#S3=Wj5vjln_hGT^T5=;H z5po;@;+AY_!s{ngcdY2T82$lrHK?9RAqUq3zoQ>K#I@~R9hLWqPsl9IY*=;uMzm!C zj=u0NjoRPQvOYtOXFvYuxXgF+3K`9dxNA5nkNRRN3_@qf}8mx-RndK(#gZCyTmKXrb&Skx| zO?|_=qHjZUg~XHX$g~JQvH6-$!tyiB%}>H~O3-V-?uhCMV0PyI=GDgyB(eADBif`L zd45$62nBt^F2d2A-U{+gfH#meS^xB|c1o^7F1dmPG?$3<6}J26A|+_DQZ#{}5N~nj z@I4uBU(y9Xb33dNy431cwh7c;ij&^pAH0TGHMEWo#lTign_S;4ohD^g&`XBW3}yVO z-R!o_(X|d-y9@%M$@&tAF6%Q-W>@Su#?g^^6=7E9ztGI}B0o0mw}qTu%A8(_-D2aa z2=l-E#(z_-3C`kH_~zJIZ=qT1MIWh7|C(D8WHtW_YhEw+LVNr-XHAe*`Y){XY?05M z^WEfQe(0VM&w0fSseB{H>1#UnH6eL$bX1z-`{+sK=lwmdj0H%|-QS1>>_pptRe6^l z3A_;96w@`4GS3rn3rrGzXTyy`sYlSb!is5-tQh|b7XQt^T1eu9PxCwH!n9ywSx=iM zS@sZfrekJl{KD*ljsdsA#3!qkl~FfMel4@gsZTH0O9ChZx*AqFR!+L$JCH$$ zqbpsD9tqb7b0fp4z)0f-(ig2E9J}Ejzo~p6Q~2!?$+_}yi^2l+)|xn_1Q6M~p--T_S`oGKC?rGP21DMi!KP|| z8#23;0A=GYHnUr5?NikFjFxx_l%L5gALe)b8HU0&SBeDFZR%q8t4eomf~+(GMeph zL!_n>VdM4=$LZZnY*SzZH8c$I$A_9`SnwNY1j%B{uwqJ{e$bpLp>D$Z9X84yYlk`p z;)x=3TJc5&NqPU9g{xG!Ov#e;Za4?6eq|sJO}Z50a1!8ha1>vxhzW->E$DZN&IJrW zQ9vpI_DKNp!DqIG9ShiuT$lPp zgj_*q|zke`>OgccZ?Sq5|+#fHLAdi>f6@7Y~DFL)3)n7i?~g zxHMbhUCGu7B%Q^ZbKnj^T+HI2s+jq3PzXGO^m&5Wjhe5Tj z0o^@({M@rYRP2v=65xQU7zSWu9^V43;1Ars1FG8ezxKN12cn@pZCbw-P0y-m!RqB_ z^p2ud8AR~LN3GWU!2h>#n~`x=a^}ErC8MW z2I7^sj|9ud|Ml_r@4jJQU;x!xoF-Y~KHy_I{b+|bkA^A$J>2}%0{8%_J5qwN6&OK5 z--w|@WJ=Ld4B51Zvexn?Anw9OUI3WrT^$ST-_ zC6Kv08KJPECZ>}~<&@XP5m{1{(Pi6B2o2dgMQh;_`nzV84G1|60_sAZNVHi|bxn92 zAu;twhwl<*ncO~8yvda%*qIj;wk54jO|ibb&rU}W8S+se2n8Xg>*6>V>E7PcgX6(H z49hq+H6+nU6cij~^z++&cVLO-ggN%*kFmH>_e!(}0l}$xCzD#Ro-eHAcSy2Gt_;y5 z;L>DvI^zY_Q4i6GQu8wJ;h#{izl9{uSd@F9WF-CPB2_+c< z&Mz?43`Jo5tpyd{_pXXRC(KYVlD04UXrJn>zVqmbH|qiL_c-_&3P$<>iunXx?^fYyR*kP11r|L5^^;#N4p> z9O6WCBlKobM<#u}vdeNsCrbvLu)-W^6B${NBw{9>_P}#4aZz9_5WKms=uLP>x$$tT zkduPR)G{7gSJpxpE5tu+hD=B{HO zjN=I%|v1iaRHRaFRqsQ(dZ#RT>w5R>3pY*r&2W?Yf zy8;fqpx1&gvWkBtbKCeQ_O6iloIQUAvFMAABZg~BLt9;j^cFe44(3~>obI@a66IL| z_Ilygw6y1T@XK?Pi>v6I0J7Xr6+;V~BTJxk;zUBr9^af9sLzIU%-T%~egfwsxlQip z$}Nl8aqdA$#a8U?$%>vZerdh^E!RH zvTxJ_R~3J1;)`EU;r_R|x*V88!JB`c5P?%QwJVUcW4?ETpgxZd*_0SB@0T(JJrPl~ zjDyzKl{!*DP}tpVkZ)djfUza!<5YK|CzYVBi70F6gLcXJq@CDPk=2|{Te7~8SW!#R zVs)l9!Yn~s)9#`pVvsU>qLzM9&FCS{y`#kZ3-Ut+RIgYT@9~v%FeG;pQ&3(R_Gwbj zx{G#V580#VMdhbjTe4GWd8G+-m;JKMW`o{>nOLZBx2(YU%Xtgk53fGSy~{LNgtO0) z+`<4+;Irror^-2E5*WL7PytLtz;|gtoZBOGmB$h5w{-K8L+$bL6prN>S z^ik0^*rLlv#%$|I78B-%F77 zK&+>YU`O`7+2)TylMv1e$2Dg?IKyP=il>9o&j<_=5DWnM@LT8rQM`GH9)=oI7%QPo zr~)YEHYBB=Ae)EpvXwz=gj+C~5pMZnliKFxWdD9}fsm+MAxr>dFFeC{6zu_6#^6)^ z(*8e6?5@yk>@k^v2rN)o-3x?-1NK1ufyR(2I*QE0ZuqXt`7mq**SSBG{!j}sPEwt6 zynWu=a9tW(JhCmY^6|Htt7Sk_9G>80 z^aGH`i9VTAqF=9+*dSboksw!lq+^XJ9f9(Z4!Q#^$R!t_smD?1!?RSA+Smou=YD&9 z$YRx)n&i4aULi|*w$%qIlmc=_23_B>2&~PQe!Kea+sSJRLY?^_&9e>QiqJ zyhEBpU$0e0A{FIFj%z>6?4CZpzDGARKl}G@ExrYDbMl}2adPzY4B;RwEY2tmpA{M2 z%g+phz9}FEEbx}<-(}lV$?J#{v^d5Y^T4h06Tu}?xDLD)@dP#739C3Ma(u2XUNeRw z_OKqK1fQ^rD4e*RJ9};foWgkT#ZE$H)}(i?{o(p?htks>9rrjH=OfOqN(7PeYlH(l zy_PuV=NAvUlNW&qco#-VN4R7i_i;%c-S;;Z_($Vf;Rf#`8sL3_I}-iWqHgS;Dy)|O_DhVq-1;sM6$@~IlAFfZ}f65goxVy zibqb4_PkTM&FAtKL|g96^6!>RF@Ys-(eo-vCO#ur9pF)ztyJ%WtDXp})g+yy&Lr;7TQIbh%8BctvjMq6be(GN<%Gf_ZGB zC~gk{mZDW!c(h(eQM-F$`vu`$6_h6S33NAauC3=@GCspp8jShKzRrBP#9=naQQP(> zI(EwTQbZM#loq@a@5p|DPCWTbZc~yYn*OLOLeN1VN}WLrrOqm|JQ^?9E)SWint9+}>A$_qm2DV@<#WowTG#ET2Olz6I$DqAQzBNRwg zf$Lx-Z3@USD(v{BLs1h4VCF3_`JgjojtE&yQK|0XLMZ+)%ft_;I?yuL%0<@j81=Pi z1O6)Q4aaJx%Bg{k7dtca7$p;1)NhlQo47Q~g`)1Z&&=D`@rb~G-N^Z-n4dQ1(0S@` zu-th(OG$=WS$WjcCUM_xO&J;(`$&hW+&sLQsGlHq<*6r0qc3&)QSD9Qb9%T}8@WIP z>%6&6a5bjX)-ji-rzYwuvnmgtTbr+G4VlfW`r3~x17&h{yGR;r{0thH7GB}p7(1>mmdWsdztzBci)!`D22$u#+;(J zvY2eVzuTBOu(jbJdLLo-B_;aA0=H`hvP4AK4ha|zX~1oX7wGA#oZ6*3bpIl1BR*A8 zqzmnb}LSy8++(AE;@!(MCl=nPsaWs4dk9~M*=o}Dg$d$H)*RulKI9N{Y(|Q z^aMJt4jy4fCUXV}hl&(gtyIGtHp(Zabt5~%I%uT*1(MJ(&E*>6D^w)}$07^bKh>KI zQtN+9{z?Eh6-e7Gk?qYwswe?c+rUeLQ-uIi1%@go*x02Vx@5+tWL2QUuLRwa`Cdbn zuLP_x*&gKSH;;Vr?hU~uKHapqky0Wex=bv(oRB~vvZgnIhOszM5@hu}5W2LZ9wdu8 zA@tfNH(~ktU$WZmA+18&5V}5cj9YG~w|0!-ZeVkMJ%du>}yO*hSzb%ps@<*X8H4 z_4T^72uhrbbxx49Zp#HdUQAg_5Av9|bYV|grcDF>NAS@!_s>iLezbmPuiujZXMcCEqQ`#QL$9n%0C~o*+%Iv9fK7v! zbluTVqufqI)6q>nPtA35t|Kp}*Q_FpEBo&ZBlycJ=8t@Sii`^jqs1L&&ftJdq+@`j z?Y^*}u`X3*evh~P&C+4Sfs#~IMc~VlK4N7837-452Ur{ps6Qf~V5y#(r$~t|w-(3@CKezA(WN$o4%%`)g&5B>v{>%;lT2;N3W2cro1MYxN43p{TXS$qj6H@qhe zd5MIihC_UvcX<%L<$NBhd96hdvoaLh_vUYf`_~2I`X=`B(=I#a6I*i?%Hzd8>f$mE zo{C3KT*6>mbN4E!ea8dKqOe%;ObRIzUyr(;frT$Q*Ymz%cC9XDok;~3qgvm>G(>Vx zm=qrbLHVa9o8XLVuuyU~++tJ-ODm_ZK~jS?uYOs_##mwSLwh7kGmlfWcubz81TKm5 z3G$ve(iE7*G{Wi(Zi9l~LQW)-m#~P7%mc&morX(`()^Ss~jqk1zY^(U-P@_h)=A0$Ut_b0GtA44yum&s#Zqrnl})5nKP@!=)}w-Ib;M@nC2~(p4d! zQ!7IMb^WH7`cAu>hdP1b=m$c#>72_`Yi*1$t5J&vM6DAQT)o8!*4kaWQx$KoU70&h zjuTt6VArbnRM0vA*Kr+X!KRq{l3<;xR2}+JfMRV}`H-rz4?c4Ckdd|2t#QT8mVR8K}VxtRu0B$opZvAuP@N@p6QApwH`G7pVoP3M<0E4`|{ENr@Y)|VPz|2ge08|xX zu>Czz#qpTFfOV!e!~4t$a8BpU%o}UN-RKvG1Vcaw)i|lGNW8!A6H{t#jhC{v7dWJ` zg5;sG*;-4LlfbWccBT_PAl5vnOb`tt_@_6pPCdU#>EI-x5#hnLxRxX!_n@cj2WZzp zv-q#jzj>9&%aJfUB8K$UP z=M#~HlZuvK{p*CTV1I2UV#rttwkqJ`Ll529 zZn5Bhbe$7O=~5@_klR6Vz*V>Zo7v@wb5Y@10J}KvO=|pz#v?uuw@RDWZh5#We0i#) zxAWR%2=sj=gc^LsYT*0sXy({52;Z)BdKbh>E0A>uQquGL=`J5{4>&k`pgo)rfHEj* zQ7mws^ENsJt+;pPh=(~AbplA7rE3zzI-&(~&*_c-i4TmrDezX}OqP}e7WHpjBPowh z0u>&q?A`)(F+2YR7uHj*dXXC+*}|}yZqJFRmRJBx?K-H3355!<;ve~Rr=|Ud;!vEZ zdrlD7VG5YvfKw)ih4jfl!=UC)qGF>Yn`K$j(b?|y@?$V6URCtbC} zY%X|7y6>f&*9Ov&Db#rm{rhyNKWJN;9GdW^30Qe{h3-Vo?Uchu$8_+jkr|7>8-8J7d(_NhuFQ-rN@{ zE=F_pot}@#iV%I^Ey4vu0h69O$f}Bpv{~HbSwhMRPX`Go8-m%HqB8~gLn9A3OQs(p z<=hEindS2P!H*V9f}!lHPV%k@xsC4BF}xoVFN9P{Thr{`L31Upt>@~y#I?TEk(_fC zjJi-foA@K#bvyov!CwOf;82BRtGfcK&oze!RuK=l&Cl0ISh@$4hYV)M1NJ9WdA|h( z$KQzp1)v(@LVz4d6h76NO^S{rN8d1<#YGU6V4t>l1Ja24pPmYXL%7uv^FPTIFiq2i z{S3{G7r)x_WRGW>C|vut-u!ajc-sEn$7-`3EX=+N$~Ipfm-_LnD+G1BI^N7+;%T~d1tjkMBxDYccvy&i@i`CHyd{M9nmHFBAoVNq9 z(%u)^4)LdvR{g%1m+g8Y8P%@AQ}f&2k4&(jU8GfH4RQHEB6?kbat4j|Z?hHZ{zV)# z_O?{j*T(%yDy;sZ5vNn$A68$ZPR4H3a~+PN@fp&dn9l9@rMezQs%FdVSRE@CHb;CC z$IZw9vSZD21&2uW$4G;&*znr8yzDmQd~m~ga3-aFbQ>(8z76R%ToYha?}ZWH(rpcn z!yF{YE>%YJJUIkJ;a^#ywu;XGgr-)aY^08n4YeidRuuF(E0NYnHQqrAGCRubp{)Is zR>D5a4n}bvcumrs&a*0xkgHKGNTHLHsOH0si;)FYo}3PMYmLz?o{uaSfEO)IJjBf7 z57S?x2VPZu$4|YRy409N_d*f4+&Ki5{yVt4?xN5ZJ%jt^_QWdZHu-929h1|?W>>`@ zS4dfs1{1c>Gy$q2m81oUER8XlUy|YZ6`=rIX|8n@8gd+2Y=PlNga$xT5=3gU=5MmA za|YiteiZX$?x>riCW05y*Ef-g%2`QGYaa1NGeTNq0au1e6Y**Htz~@GUh|hfj2H{- z!+X*#I#*>=f{ScpWrLbTY+EWKa)IrLgDso?W%bjR;L&|a+h7iT#h_L30h&(gP}LWB zLLxW29hiR)W-6F4yIAgA$ia!M0B~MYY-3y?x?DC;i%KBO(lRV=9mIL;4I zW@%!q0+?It+Y~+C^9EtwKe|}+$1#Kn2K%pdxCihGH&lk%N=5feoitLXx?Md{6 zu!!eda^pwMpI}6q#~rr-EPPk+M_8cK9v?CwoO4V{m%7QGM3B5eS+~>lQ4pg;UKE6k zu4_S<_KMOIoZ)>sr;I8Y8OIp`(>blrskRFNUJy4VKiFncB2?mc7eN+Sd%85D?wh@F zMSUG?=xJ9xJz2xSRHycOQ`i!Z*Rob33VBaxtP8JL-W7gsYgk?1UzHdGdhgcr)glk* z#q6G0r{~`u?QJy-2+@weLAa=_h{;C7vLp4>Sv( zHEIxRpRE_G#p^M8b)M!>dSR`4TgmI!yb)!dPHPGgC&UpK^3DwIGFL5ud+y=Kvwr{V zys&=63t4cNj&osxJF14ggbTg5+PXP~Q4@*6jBB@U@`}UGs_p*Kg_Uw{`if%OPMcS1 zlNu&d(C~|t9SyMhsrjCHJsElUv>MTjH2@-7sPojM36Yw997JJRkWvs!uM1I1GfEm* zA>S;k5ET=08c@!K*|O0rZ9N zsy_28KnKX;CH0OUsKB3T=-UdEDQC%`v#+mVc&cLC*(pcuLyFy6nZ|aYB5wjaxT!PX zqDy`|q9N8WKQ-c8#DBj#kh9aL`(Y&h>a2M#U6xVXx@h@kwlwSK=>qib`@N0Fu5jK|v+B^oK{)E@4U)NFx5H(PL>o>Jt}&plW&(|0b!WIc{?0YH9)Nl_fSiq=+h} zqQ@4GjadBtG4jH0uN{Pn>T-(~MySiy#!J(={?PqB>$2n+x~eFrT=JXwrW;XlsqP}- zc$AV>KRJxjqH-g&3%Ad*g`Cu0S<&6tIBe%Pi05@{5xs!`%{^C)>qRO}`BKP^Er8Eb ze+Sq6djR7*&$k7sVicg=DBfcjl#b`+*8=19zg@+C2j_S}qGTl-$RjTpjYxOEP31?s zQC%{Eky&@z4j$zb^p9!V)Y~7DtMjAR)ZB3SvDkM6x|e&^zM2%qu)V>oSfhqJREItl z_Jl?ypk8gtXdgM%ka>SI$6T|^KZboHoZ5%ALN{zob4Uqv0sPKC$E1>tm3coZ0Nztx z$I`_y^LX*HpFH1h>=@YOJJSj5dcm`L(z}Q~{}(Q6=Uttc;2b+9M;@%p&@{yY4qBti z&Wux$TxDQYT~XGiDIpx9vUVolOci(S-yDi#{R@MYk+Z;mc*jUQ?Ed0&mp+kpTrEk= zInEx z28(P|Ty{UH?#dLdS1z;pid;xf!@mwtTpL_P&^LKq9pZmV#B7Us=^Kzdq zjNe`<03C()f7ce0{-KBC?(9d{Rh89C_SP6Xq8PqHo_v2MtYH|8!Uk+HGpca_ZoWX%LrBYta(f6m7?=NryO1F&=S z^7P?Y?gcF@FyBAVB=UIf1$iDMCBJP29xOCb-%6NuB0lkiCKB*IIu(8TC^6olx21Bi zdeM{LQQPdU;sw2$=LghM5nMy~`4CkpfYNwTRkbdC&rA+0fktL2$u*$uGgYNK;dZ z9<$Se>rifh=NfsiC|Z{qJ<1H)Y!k-N|WL?pka4q;o9FuB}u740F9-avwNJJXhjS&%zx9Z`!h&27n2`zKI;iD&PW%hWg zkC@n}z=+*=i^yz?axXV0HSqEC{clzzFVs@o7ER zOhYq7q-qNkMdI!F5Ivbh_^d)nSk}yA2(IOs3gyPleDhhB^?Zw~>HCT~f?Ee2MIeY- z9+C5P#h=#whokFCrZi+?xPCZ^Hv$mY`==5mro!;jkTBB5eFJeODjVmA1u1`D=+vT{ zNEqN`mEmbeOOH`(Ihf3-7AIUMCZ1Rc6!$*nkbbllH}0_0-2{emtVTsR?PsB?E`&JE z?U8C8&h}(vrr{`1sad>0G~j$<8*xXL!J{ZzFEVQy%gF>UvM9QTUDtK7^s&fkU}n1^ zqysTwB6r6zSuCNUiO6VNVV^J|B%ZHm7_265aMbiO7wMgg)WWdaAlUtc^CUp1-Bb7m z$B*5$og$3jx90FVYOgxGF~l5TX95D%f5? z53)n{2>zc&#UyY!5o(rBesmFSbNu{7b@9yXUjH1_X%RZbFb<6|x#X6UZ!!`2vPyX||(&hd2Et>hsz>VGI z8r1{wP)jvFPc>h~*XtE_%9M@_>U%v*>}GLxo*Nz>m@%VC>S~}7_Q3*KE?T23Jhanp z9f$}7_y16y6o51t(kT2G@xN*#B6Q}|ku))R1mBdlDLBVoh>(~cvwvNN#4l-JW*kKX zY(;?{|9>y^II5Pb>_($$QrKoc-Y^^O`D)ZMRv9CaAO+ktMksEyf=29}C z$m-*mVVHZ!@x=K?Ycc*gw0PJei8=HcQFEIijBAJj*Ut$=_6N&gCImirITXTPuu{|v zQARVFmT@r!$vO_~O#=-1-;$ z4CSIEablV9^*P$DT(w!-I~Pw1bL74uGymsr#f&5?BF5Qzr-Od7{9j*S!ypf(_xFOj zaPR~w8BQ-*o79dukbvxP@7^!?{>IbL(y3Rv?#>3;Ej?Elv|#gg>Y*-RVW8$jiVH$s z~@q^WPqhyz+^tJwCc2=yJg$KR0cFpHL)F z^r+@(G0H>7AtJ#I_FhF;HS!QPwyHm;i(RSG&Yh7-Hm69&P8Ee^LK!rRVGh5) zAEEu;^=X93hOcRzzH^7D3Q9yV5Y?b&hf5Fikth@C)vr*dNR&w-z{F>t^BEDz{ZuT- z>928o_eX9;n5#iD2ux-Rm|(gBNrT`->$mQsTF>;(usL7wK#B*LIJ9@^PEGa}87J*n zpR5rmJbD6qd=JPS zp@4TC7q!g^jVk=LPCxpBZa7ZL=1v=XyaD7{e8kiW%h%=KOUavcAS`Kc8Ahi|KUNm4C5Y&jqEte(ET;`c7jYytRH0=R-#VzsWsP!N2a z$mCz?G^UC``+X4{IY=WL;=71LG3wtA+J`#G&S@3ml})|&q5Mi3v%AVkPR<=ltor6e z%siPG97ZopM{JN{w@uune$LdXZq;ThG*g#oupATfv}8Z_d&%~DQ3jjp_;Z;9MN1RX zz6gGQ^&zU6Z+ejKM^Ld5u8g;9b0>l~>6nR$Bp0Ql)I{0@8tDU(%wYlp8x zMvm_8{&(uYgNuW^1NzzRL9mw?@i%qAX>ru-RB2Hw8Pm4Z6obY$nx-9XZ=1FcXgj9iBMe5 z7w!}|%{o4|y7@O|1r|?qvWI4pJw|80SGhkPqL=XN<$^I$CxbWe3K#ssU63I2#P((y zIz%S|J_gA@1swHGX&zQ}82ra91i8t^foKF8L5GLtjF=8qj`p719#21d=Km-Ckx14x z!NYMKM?Kr=^R$QS3b}TNa=R||b0h*tQizCwqo*%bX>yJ9u!FjhZaU}<+(%LVKOvC3 z{~H2v!RHVu=$eVn-`NtaeNs}mj8==k)`_7_=yrUDHu*vZK8iV>)94R2oowzf-wk@tJflI?# zQn5)}$*kH;*OHiRXjCSHYNVBbMcO$jk5+#}@^1y_Hs`NvF|0Rg0&jy$#B%eJ=T)c} z&5tAvL#4Uw7TN=Q9NVe&9vfQJ#4vQr-sgaE82Hf$a?RC1+HkGE$4X2u<>R1;F!5iF zJt9I`qn6~A>L0~%^OKzC)0{sHqTGLYjjU)!KuO_xXiwfdpP zgGimpn(Q*uuHs}l?T}bE4FqL(FJb#~rWWA6z3O#@kIT&kOEozoVNc0k*=PM>y4Lk` z2>xMVdO>woaN!A0su80>>6_=EO>mJYbpZnJjZ<|j1_G8V*<4_N6jD_2rV#os3(r9| zP@Z8t3PdqTg9lLz_-e+dXW!<*8RNC8k*z*WXQO~TXI)k;dmSO`yHZhdV%7AazLJZg zb9d^M2%0n89j2=GDjf2U3TO@RT^(lN6~QyM7JKeOg6dx)ZhIH5%3tdTTAg5n|5lTJ zqmS3kMaZ&u<(!OQ7o()N9iX3cHt7ABqgUVZ7(~=#-JU~mZNZ_r&)M9|xZjlDL*_{c zj5l)=egSm>{#2Ga$RhLv9zkhGr+#Ab3M;Egi+Szp5+KvdP>oY!55HmCxliewxr!PqnaFJfA?gK$s$Jbi03- zW?|uFWgG_M^7VB#k1mLw|3NK2giiSXYe3;%a8=OfD{ql+@YBO{{Ylc7mmA^;&ef0M zV!$p&dX!beXefak_bPqCIXE(N@>RrK0qem41Cm;p^`IIgaWD^R#t$f%c3!+yihlW0 zvkFO_-EmBZ-om{$k&xz5{VeK~^qQ;KsIXm@msVHTG5wHa{gNkc5hZRFMRV577L5L@ zTS{OYfYBy?nh7Dc4?o|A0gm?-Dc7xMk>u?;Q(B{)Hg$$G>Zy_k#~-JzF{j1~ef7W3 z`jfF*uwsRRZl9eqY|k^yVWN@Ihn72vouHX()(YPT@gA%m=1$1@9?3A7qBk-!=s1w= zFvD{#MR(cLUyxmI0kcSjRL;DF)QKjT(dHi6jeSAB!Q?6!v*_DX(RN)WcMr>um9U() zcx4r=(!r++nFVNWVc#7no7ZlyRN7$Z8(9#Js!^=`MCc+_7QPO&6POFo z4{Yaj`r>S%Mb|M6r`L$jp`#RjSTHV$bZ5>Y1SPIpKZ>}`DXCpGMJ1HF`q+pW>|KVJ zD*gPef!$b7%J-P7b%G;ygzNPkRtNh>3|{42s_2;jYO+4S}Gf>+Y@Lynait^gJ96>yrs+9p;_y;5zw&>lX6^iTqQwk)B<*&~1 zI3{Sjm6iPM8U#C=Ca#eA26^7)hwj%FaNL;Z+XLP+ zDraJL3^CORORGh*?B*5jo9SS?*h&QDvUCe`(K5pzSu=)zyvlJ);aVNpepunEQMdK? zVM%|JEN-Mw7tetz{cm%8_D{q1NiiV~y~XLK80#GegsTW;y$ZaY4%_}BI#B$&fr&v08^z z0{R%pJy&3LMu(uL5U|A!`|QmOo`jrV=% z^BDRAJlJ>=pixnemrQB;K5om3W3pTX2L_1512b#nYwVND4^CHRxwG?mOi#S%x$vLe zq_N1~;~~+K?3PLlJ;W~SmD)}v`nC$qWrBDEN}O14SD5Niwg`JHw#t+qpuql5t|+-T zgOJi<)R^tpLENhuEqfIt(;FyYbUE@Y(<$>%=K9`R&QJA_ANlIa+rez(CrP1}x@zP8 zRD_peoh{6e{abmfWfXK6P7CN0O48jUebxTqFzJy{tPzU;x)SvQ4Qr1&$FKb;{+jfmMocjS@8E0PjJE-$3hIry+i{IuZ-;%&L76DBm*T|Z9$ zQ!f9mgIQ#cS5UfV#%{VvFQWQup?>AwfFvh#iXEwX7Dr9BqTP198pBDoaz&n4bgxZo zcxMeP$84^-A}ZbrloA&=4?}L=g@8%+Q3U1N-CWhMfkJTW+8ChD;Wu)Ntrj z5{K=t8&OI0y1aZofY3LwBUxLcUob52MN?t|ev8z_-##e1$1H2L&LtsMo%Qr6a4$kWL5byY1?@yPP7m8<*U@ zEp2~BgY>;oX#!y^Q+PNN8|r3?vx8{J*h;>#1+(&S43+o7PmC4;=JgOt^41DCwn%K* z8Y^v*$paj<{esd23MB4uXi#SXa2$t*QFV*Wmn&#U92~TVZSzVIzQv*%G_ zJEXb}Gg+a;ZEna{sz!0|o?PfL3O!baC4w}$3s>P!%HNXet*|A_V^jScvvWnuTBO|2y-@iH(fv`g-CZlR< z2I32SBB~6N;Ki8R6Yt!<$fG|G6SM`tqva(h{^_8ifJ4{DkBOM+Fk3{b-((1t8a#%< z!>g~H0#lxgvgJZgRVpqxsmnisbcU5`$yVZ#6b>aIaWPUhhQ^P{1ed%WbER6Z@=*$m zIjGU1+2S3eHYrJBM4Zd1+f=`z%v)EkrXOKgf-J>#NwJtTnT=vfO$$!US5SuzYPa!H zbm^Wq5((n#w`-YoXop87=lGVFAvv(D(Ey@-G@^}-luWM$6TpLY=h%Xo;S-tJ*&%Ak zeZWu0V+P4I%tWUo7B_$q6%Ri$NAbRa%h%aX)!I?Q+saqrWoqP$2}EQ-rZ1^A;o{O^ z({>uU+7sD6#(^(EOE0mE#Sg)2xTC5i3i7HZsd>xH{5C`N3>v{0^z5&W(+?&~Ze8Bn z=sji6)(;q|k-POy+4+rpQ}GX`uLdOczCd(oR*;I}FP<($BrK`SDV!lwvuAVilC_~^ zif0m5a}w4p-(a)KCA#qkF6#8EC)bLfqarPmF>nS4&~mX9*u6E&{CNP=PdyHJ8zb;! z^EifrhEA)HlbbllA-ZAdJCYFb$&LtAOc22ok>o8o-o|RKxX9V<9ihHpL)H&O->w^k z=5*N`>nS1*e+--na^!=~3x;~n6ZMW*u4IC~A4#IG3tLw6Sl_=zals;&RS2f@?B&z! z)$S9i`6=wxa>F-=MFv~E14{EJFQ zikFKXIz3+v*(l(U9Pi#JIou@5Ml9iu(lpL0K~Ng$$ik@H^OTbyLB@LCeae_DxxM)Y z=fdcV{m8|>v&70WS&c;B6XJ8Sf9yM-;JocH#%`E)HVh zAG~=F%qtc}nsL~%I^m%@^&Zgy8kPCOe}elMD0hO(&)Le8A_kCdF)43({+}ohMO5^@z;?N*6k0Y-U782bU#QMVTZ9+ z^80Fdjc|SmEe*D_{Q7=LASVmd)>u!_QVre(=DU$)oE2M1HnMRSmqhGfjGD?wNsB(Bg3Cr+i>g&Q zNOS9Le6EfmlTykgSt-q+`W?)SH;N;M2s?(>;~pfM2jk|xwv3^Nat2gm^+wqb{<%Xn z@kmYgh)?ojF0H}q!*RbZh^_uP8}9Uo$g+L^E9MWUUJxECuiKPz)T>3I`s#ar&>IVf z+mw2V-}TJj+Egt=v;z(_r>OF|xu9=h+@KWB0l-h9tOu261@22tG?smeC1sKKFDPqS z{LzGFdiHBc8b|pGykdNs+vijrhL&b~@>3BwQ_a!gj}D-RWNVX||e{GoPZ;Pd_-xnh*3_=9Bz!})kS zFO#rNs{Kd=f9J<79k@f+btO1cQ+bx1qPg^3*rGAG?p)V$?LX_Y_7~or8c8>NC~o^- zSQd{ib@W33N4u{9s0|t)ukOxiNAo%z$AKGsM1~*nh2n{QaIR%6GeJWvk$TB5Oi~

FN#!2v~Pfc1*xsil^$DtiXYF}I{xz~^%hFo##$SFeMc z4q=k-T_soTsu91~Hlol%+(QtesbbwxLX)J2uQ6O~C7X2JEh4&2&dO~vaepnLsJKx$ z&C;hMcx&h41-t{>%SaG}t)%X`1}SO3Tjs8x-*GH0??8j?^Im}udv_lP|BsaRaq+PC zKT_IH&0WpUl8tfnl?17}uiI!#pafIl9Sn9ww7|^F?fni^)`#GvAltC-9tuc0i@W|H zD`eV>p!A}BW%M7kcZhjyXpx|sIf808g{s?l{AE>G+sBUs<#i$bLY`*?BMG{uj+xN} zVdJ80P@#zN*ZuYJc>I&k9mzpWyjJb2}F6MACJ#t*+g z+sU6Ra(}fJZ90ooJSYsJda1wupDNVp<^Qchoqxs)OZ3Th0rpNll|Bw_-Y$uiqZm_^ z;tOm8$-}aJBo5uuF$0&YZ2!XheA@S@&*r#aeO?9qZSN>8csB)1Hm8VaOMN|qEtQt;g8{sTHY zupOnBy0DG#Wp$v=lWvM#paDVSUCcj-Jn*t^+n{LGS2X^XFRKs|(W(uVQ8a6n`cEaH zG)~;Ee@QZ<4UO&Kg0=39qR@(pMli9U+7nbi*f&t^N!uOjl3lKRXUKQTO|GqxD5F8D z%C6oXrv8-6mjP`-R$|~_e|W8 z^^HjT6C%KF9SsE!ov)-$qlRU5Y%614HcGZg1U@&D2u=&A8}fA-FKyasNWCCh;8*>$ z7CYTUULpcZ&XJD@AM(XuBm(Q(ZaDo1hn+Nd$#_X8#$5bB0AR&Y(v1SlV+w=mgI$-! zj~kSbm7o_8O-4o=c+{O2Os^!m=kbZ5O>NViq8PPpjYPKtD^X zS|x?ZD2ZkI9t{p*#B64PDa^%&u0};9~DC;x?{ZaSnEe?HmXxH{V|v^_yMW zf|zXCM%&G34_#!FI>eHgClshpG)#VGa6x|aPUtp^_Bz#&&4x0&I5i*CmIWxLeZl1G z{`6$Efe0^IQy7O(i!#xnpIs=yCTu9lVmyLGT=dU?GOoNxEsJh`wdAAN&cOSIL-D4>|r~;1#!CP#X#ysoW$hk_nDouOa zf6y7IRG6v4iBmAj;F0L?_fS5gFLZ1?KdRe5LVgeL*L<7!r&t@2ZLTCY_-NgNKQ^9Y zCR?wVZbtGRLsO&u60so}tP&;B{KY`JmI@&R<%#oL59LTr2L1Ui-; zzF3-J!47;z%#b~y#b#^WnS6<;VR!5Rx?pmqHE$qE{r6?6mZrgBJyg${olxXoCh7*k zVIzD{(awd^2WFU|`!=Blo4mt6;Uedg%^6>64I4UIfw(qaA%MVb=CUe zie5zPAO&@S#C|gVwS|w$Pv!+ z30yiBaPE)Xq-b|@f(wVtU(JP?R(MczqTezV$`eh$$ureE63$mK5~6fL&JE1h)Khni zYoIlC8nj_c16)EF!V4+)sbz;dm>9B+ZgXVdskKHQ%>Q6vEBdFZqhlZbeAA^k(9jaF zW82E^ag;VjQzXjqLw=(?=4zSsTj9VeJ5@d{az5SOXnz*s9cOgDX7n9@1F{_`4Mq$2 zHRLUQ^?df-jyd%MJsf=(9XN^a#E-yl6KAA)>2$Y2l)l`|XeQO{yWcI4lQw(5d;1Z64H2MMfCkL*CsgN;0Uuaa{?B0s9DRna+EHQ-`-Gva44mEVg3&5c|x@di=n(JaDd%< z{3uEYQz8howbpAx_i;&+9QvZuC=aN9zD0Jg*l_~67?hfsro63dV8Gv;l!40zgM~f(o1XS7A5L9n@eS;FG$!DR;LXx$TL=|L zAAOv^w5HY&^Km4lVWaV>Ula~CP(4R>POWseytvX?(1nSqd@kL)d30 zjbJb`Vvwv1k5dG9l4z>iyH(xWH+;h*SMEU1FELg8dwzfnTD0_$6>2GmBXJJYp64G? zSSJzus!rod=)z161V#x65ij1%KR9f+#@>F+!^9R1C?SSqCrmT?nme&tyDDQ9W%+Ce zWKvdCPm7i!!Kw%M&x1d(GpL(mqhc6xB$gwomQAtfEa!s9A>_(qHMN>T;w;o%tQain zt-SF}Od|6Xsa8jPt;3j}>aB>`jEbdyJr>Fao#=Vuz6zu;MPA#;{;^n|ql_&{-RJ#| z;9q^whvl^5V+YxQ^DV7Vua90bU$r+2TKx5`>Aw1Uiu+b=69`C&xxw!k4Q%|wnr{RI zxVmO;0!dI5VJ^P_7>ai9wdj#6lGmF%Fa}MVZmce&f`t>84*ROOg2PYzLOYb)yBVoA zzkqv=%c}{;kaUPcn)Sx1LGDC2sOC>cw^0>D-d9_UMSYL%j**Xx2SQB9d~LDx8acJL zm&NGhDXWTD7ew`8El0?E?UNey<6u#->*klD98D_(qdiplzr0y~Fbn;`pM9|?C`JdS z>-bbrTL_(>030}zb(n#gK+jbYs(V0APUA6NSL~JEt9&EEyROPVixum?G zl)9pREaoX+;g$iGrFt!PBRN$iaMCoxGTyIiicb9QTcl7Go9+H5eE^2@I08d36NZRO zSvNUOw7ddVlDsv{A}2lUF{K|>OEuFS){!xmAG#z1ytXz_csjoY-|KzRlkN;S>onS5 zd-6^10O;fGoH+}OhbbGW2Cb~J=C4|~Jw1$&X*}Kh=kxi;Md0SJu!OJqdFhT={M!7# zf29QWyR2myPmlTy^`wVaSUW+5?LR-J(0^!oOiv( z=XBkOQ~S*P7J+l8~?$>(Od1nWSkDkc}yZ3$`-ZWe@KM`~1G|#XoeSRgKipOy)A0 zv&Q_lxBD@a2@KN`kj-c@5=k1`;!=#Hulw-Q8sVR`1c5f8()dY54mEgVWv-4kky_&jYC5@zJ+jM9;8i7t|&l(0CfAok3&w9Dwx;^_~vuvlh7 z-pY(2%-bsT5))=@+!&@F&YafZyO@^DmT55WcAWK_wJm8&^$Hx?>SSlmo|6f7eLT5%kq$c&9c)?LqA5*(S-n}i z$cgIYQEo(CB#pJCSp9*|PewFyG7Fk2)p_ST*G6VwUM61a_X^C5Ji68d+27Nov>;H1|chZrE^+x*}@v}cQ4c`LlR=ofASagE!nbKy~^pbvwUX;e(};C zyn0Rf6rwztq2{O>M7@sZ9-rusR}xY>SDuSrczlt>&+8MjgcM+z-Pi9af%7+j!FTTx zxPG>0xbm42LpKFXlX>Hxfl9KG!8PCo1j>(^)W2d-6PNBe6Xb$T0)NI{M8Y|3%2!7> zRY%hzo$?BS%_N?D_8+gZLuM5qm|F{0E#<6#e%(O}jAYFv%dF01Y9>FlN2nU{ZHn$? z5Z%=k+86{^J(OcIMKUykQ8fI+{HNU@Udx=Qpxrcd{x)AsMa= zB5#Jr>CKycsEAF{eL}r{N7ID@M=}edf7q=r!|mZ;8UMP}Tq_J}-D9i^P1t<7%b0tS z`{e^*RhBUNnx*wrStrND!Ls-onq$VYlk;u1q}M3I3FWbg#RZgEN$t5S|5Um~d&x<@ zRGF#4U1dyrYa@S-M@UU}C=c$N-NLdCq@_f#Nm$JJe5XH9U2M%;R#gFBXG()4HqW*F zKKxg}8@1)wJ#gVGUl3IFyg!`SMcJ`Dh>nynLd0`#{0>7E5)?BABTRUfdRPeeB+IZL zp%61|KKWv<*i|BC_&U16(Xrx+c||vxQP8R!S~i?WR|`0q{yTZI;U}_>N9MthOqpIA z#EDr0H17WbxN-I;C>K;N8l#~1hQpGM%P&4GJ25eozQ{E#q*_5*{?=q*%;Xz%xt9&Y1T zZv{X>TH?(Y`-IE6J%Bb0JSp7lk=Ls9*q}5^q79G!cX}QApEED`iNM;t;9|ayeul8& zq?~<4^_rnsLZ%M!IYDenU=$v;EA5Qj3RE5p?(5U>OL7|IRO>8x>CrmEC;i>+_&vLR zM%5p@Z?lt2np@zDjU3|~<+UEeJ@7qJyQsDJTJGRZ?4K~pFgPu7+#}Qwd%2o*+uVzC z_6{PG<30A6dcn_5G(1z-!SR==XBc5V(_KT8+2+>*kn;2OSK zctR?Ks8OQ&Ic=ur>uWtLgVa&QWJ$qJ7j@>5{y@dc0lvE;V@T-)>AQ9iXSh?}ou6~I zr))?ZS%GXa!`ZG&Vw~#)YN497yh#|gjs^?M*59qAvsa2`^s7x@TZhCxcIX1;DbA>e z|KOfx&&6+#jW^}HF%1g0d`<>v>-V>zqL2>_yK(JFqaqERX`JiY#3!W0kB}M3%^u>_ zTRqdA4m_F`fah+WQC0@S(5G35_XLN;=hJ_;yEJ_*h`cdlo#VSk{J9on2q)<|#E^`C zaDisj#Wk7#A!MJ1)SwX_Cw;F3R_1L;ptTsdh^j#8Uyu(o(c%?dG~qF0T*rvqS`JFV zeW(k}?IMcC&wQnRtPW!0$pN99D^Ji}4`N&w0sN=dx_BYY8UujOm)pDcqOU9)-1l_T zlP*zWy_)Xdf0tv-0~AeAL0{k<)@}rhT9?rc%e>lR;aO$4)N7X~;%_{A)M|{UeKI4p zG4)2S$OR*fcS>_|oowsJpDM2%AGKb5lOH;wbi_1jUxSu?_khQ)L)eJwncRPRVW;#BSdZW1(we_@QZwNP z?i=fEs9^~$POhNE8O%c~ZB9R{g7VOYg-O3ePV_k}mM2#6UHmL2_17mpsO9j0-;Nr3 zD(PYuS`;_`X#F_1gQjzUBFF83HJt8hg9Z$wJMKlA+zybOn-4wq*yakYDGq!I2l6X? zgTa0QBE==W_QbtW4=e*M$!FdyD;-#hqDB7Xli*xo-m1OmlX~s!eu|8vbw!>AOyfH7 zoCN%y@xfsUFugF2Lp#qg>|RjUzOGFORL}z6u>I4qG-MDv6iK=D!V!oho`4)BWD@4J zjFOyQRWcAO`no#>sA%cf@ z4un=|;0)B{$Q>l-lt8nvl?Fewe*w}X16n^3%CkMD=Y4~TSjo2jnf6};4qke9{&;!o zCT4WQmW+|hadJKWEebaIO-Tc!yhmo*TB!lbYbI_Y%10tNRf|*mdHtPgy_ z+N!3%cks?GRJdnNyZ8DPkn2{U;xn&}_+w5XoNf!c zYs;M@c5I=mSpd=B-qW_-#PY3r`%f1^e6V0=`uo{~iuQWP854jrZpr>_+8_P}I5#&Z z9R!T-anA;FlW74-_wB*j*lkOzK$rTj+EZ~m2{HRG=cac2T8`Tt!GioESOuaDfx?># zb?G>_#jbIJN<_f>XYr)!=e4o3r|{zZwOIF?5sGHmiS|gmB=afEf!t&oRSro4_x0bk z0^C3}Sei4FJf{<{Z+q82SW0GpR3z!+ctsT1=W|b)a*$!V9a2ZqQ+)+~P!jDXL@#2& z&g2HR2G2uE6+)^^%xn+h2h1p2ozBl=;kS5Bw%2e>dGUB}JRhUOimI znVWH;YG76V6u7>aJy@S^eYD77AN)X0UHdhH7afXN$HtsiM}0IfHYu5lJnR`Vr`T*6 z@lBrA$!2L??)qvX^T`13VF8BLMbE|av5&yZXE$sHSaNTk&44UfD44=kBpKp5dkn(pY zv5|cZLhlBz;Kfj7`1#6UjIeH3JJlVs?K`M-wx=^^kN2iNvTVxN2Gbn8l&J}V#hs-r zb!E=$604gQAfTLXBpC+)O3l!;_Q(fCEhp=#wK@<(oc@$f3qI5#@IIBbJ4wMHFuBeu zY1w!0K$l#h5$Oh$8HbqTFhJlqn_Y(+q#;EmC}W2IX5-L-d83WoSo_8dC(lIc*Jslk zlq$<$)Te73IC^1WlC-FAPZd0V(v*rZF8SmT%{m|IExcdXX-x}#y5b{2>z>#|X=ddg ze*&HHE#dp@t0Kz@l7ctLnMe+ebgzuGAoTSvfH@hu_Hml}t_<&YH2QnMMn+PB0gd&i zV@+gnN&+i7XQ6c}#_Q=&5ZUMYVl&gXYc4GKjeK`g=?m5;{R_pcx|hMc@2}e=RQo+~ zVP&60zk%B+pJl1Qj60xIlZg)J&R7Nbptz>h_d1RX|DnyR4EpWV^e^6*VXyCgng2iwWPVE=`VM&ULeT^iwz~$bPTn^j0{3BG{}wMP zVV8pq zqx<=Nz^ln)oute&&)K<|OADD3k3RYK=eU3Eq$@B*ihM1+fAaC118+jxfip=#dt08F zAT`%d5*MH!Bw8|{B=&Xi=FEGJ3Ks=|!Uya6HF6y=ElO8ZZpEP{XRtDFve&C-F9O$q z*qWNPWwlp{K;=_(w{3=~VVT~ji%_D**Or@?{P!+#-hytAZZuswadqEOytn5~xB@ZH zr?jd`@sq93W-<@nBY?h%gNj8=t7Igx0 zXRQRZ5tN*lXzn8|SmoCh-TOz2cE(vx&*)_unN(j*c=QEriI-6w-ZX7xY~-bF7gYY| zP2fe$eIM@)zYkDQzcz#c*4{@!E3SIC1ut;ah&r5eEOmKovH z;GnL}B!WEF^+cAY^uV1qI7AewAUYAc%PD3d&f_O~L%O{U(KDXAyzYt4pmmlqf)EUAOpcTy4j|_scf0NAd~X$d9&3A{-){AM zWB=vaDPc2j>l+Lj>UH~Ibmb!evpm5t^NDH_8P2rij>%~+8{>LaXHu074^VlzSkM8<^|o8 z@|3xVfrt81DW!{&==J+Y3y zcca^Ra4ld;H#KZSs<1>Bj_*Ak8=D&=2E!-~I^bTT`Z4>t^qi&{a+6mVzMAC3!Oi}m zh9cMfw@mw!Iq?OQ*?imHIbiSt+5$ai&p=95Zu9FQ?GeB$d(dHnFVUKPI7OOpMLDf7 z=D|_so>uP0;)jwjW5KcjP_Yb+z8;A3HuesHv6Qm=nu!%)1lA4!Nf>W6>0z1~V9qF! zZP~+`X6%Kg#(|UPh6vAqvVr0ckU7(Qn+N*yzZ#xNu4~K!OoN2pDR$pGVwQ^;SE>&B z6?C^!zG;C2h52Rw0sUMz-vIyqTJW{Lt$_g*>sHmJUyAgX-?3cRIJj#t%#+CEoIyxO z-W5tW=D*A-A2pC27&1pmqA-~6z7TxCVPs(Jk5;$d2?!0-=&;ZJ`DNUTVBtPtoRDxK z28g+uz6WWOSDJ3scsh$o6^(aZB19sHLIf0Pm>6%gtP@@k^V%CZ<62utsmhNr+Jun* zx>Ts(L=%{k^wIOS#WwkXMPJ#kTbmG0D&FNGr-{0UpiDB4s`1Sg#117T~9LFwU(%CNrtfoHmuQ+s$+__u zkE@Ovgi8+H#%wDSHPGu41io-RowZDGAEoXaAGi#)O5MK9!cMl^I6{?R2C&c?3>rhR zqr&U|h;mfkNjxNc-TaO)fYR+N4RZpMFeBNUbQ}3sNbMKt)sq%!3VR6phtqC{LC%zN zOn;WErHUY1y1L=0ao%0OE^jV*{SKe3%p=U+Bhq5sCRY2%37MYij_R@hEH4<*W%818 zOBc&UVM}r($GIY>;b-I58b-6?%gt~rC@dgu=Zbe^zQu&K?+!oPeW?nJXLvzJ zaqbVYgu8X@ATvslMm?e8m&|QYKEd6Jm22^Q=%?K68gSv0(4K}s<%V~CD_Sw?Hu~Tu zsa=e}n_c|PrVSSTZO{t!2P->PWbHS790Xc~F3#}3Mu?zE!9_A?h$G0I42cnz>V671 zG(R$L;bPtK)VZ;3r)>4t>C*dkDALS4mZL4?CFdU9<1NAI8iB;5^_bxkKg2SLde+=fx48oIm*EL6#T6QY>&?=sBddVmgrk}c$VI?~Bf(OLjq2J_ve zNLD>mAhiQhrC@$&(W~ET$l26Q(ORetx?WLp!MFvy54-L>pPCVnR|t>dP8c~pyv*)n zL^8LOf#lz4nLT3@8hf+I1H3%NpyiI3mycL)0=Kc}897d8vN)l{GGcTFJ7px z6O$dDXAse?yV%JVJE@H+0KPG*ruqBVBGZ^jT4gaW1&0DH6uxu^Y5S7;=UHWU?Iod& z3OMZS3i=(a?`!+YrTcSkU|u5gk!Z}~5N~T8jCEfEy`%~$1_1`EFI%rJie4LQy$Cm( zl8He_VH3>ANI|)Kk_`cgW)`LG#V>|=9MqxmM?{|U<>2Ytxhj9aa^ABxwS5x;&LoZ6 zF+q`d_~h}RCi$Y1(F|?}8<{%=8_qE~?KEdw#vw=nfP6u87PX;sgCI=yfhx_U`ghQN zDB-5qz4@`8qPlrg==u=LAdi+ud9)OvSOm?4OIRqaP!1#1yN*6ENW+Y~K3!=)P1&8H zZnwWMS&7_p#B`}?S!S;B)Skqb^e3%ei~8P+YqnF(BMq%F!!2FOd&<1Ds9xh^9;c|j zci;mwB%~exQITB>FA)kN>q-HpUwsAmHyr0)YkjrQPel3Sd5zTRi(h~U@$ElP!E*oA z5bM)bB~kON3zJKn{MSKMVNCgB%-7q4(Z7Xn@38e-a3)SN%m*FdeL~D~9cw?EC-L#7 zI{+OfN23f-#dW6$ThcFkt%1y zFfi{LFe(a6aEHcIq;JY$-`hr#$|_-G61ONTO$>9TeIP=Mxy7-Ua7S}AKQ%uUwss-F zz$vqN1_=VEN1`VbI{Bet;wG7f6x_knorqUg4&ZPHi2|o*9E9B&APe|7F9@g6%2ti> zaBEK?HT1uH5QfpFrXNy2F-L(#MSarqUmmWD!vKeaieq3ZTD-ns;&^Cbakj6HZRFz! z=7cvTbtn5>7n?~9wmw+bXu$Nm12Xk$IPsU}lhiPoVX<&|cFvjKB2J2bVhHiO1$g|{ z0PUUE(}b_lTL*N)Ouk}q5u|+2PlzbLh5#0j{jrv_#DysMfRD5{3{YiLKnitofKzpM z>x(8aD5l1O0hgUA=!FC6m8uy%Q30bD#&D>WI4-#9ap@c;ZUNAP@@kJP`*$c<{f5Xf ze_@CGDX2%RiXvgu>l;|+GG1KJT90Pvv_@SkdAqz4+IA-b;S{&)(W)Ua{uQZ>$hW`M zozwoT*dU${XcAM$(Hw>gRZT%K0C)(b{~P|Tf}_=u&U-*SXFfQFvYpu#89r2Va=yah zJZ6J8VDoc)z`V@%+zX>S)spCwIp>u5opv0^&LevOz|GBWeg$yd3m|V8w!Mx!fMo&y(Yj(vxFm@xYP3UNR9^{UHY&-if5fC^ErR?o1 z%QoooI0ONb(r@a#@CXs_vYeH1Hh(&2y6V>MA1(vjPWZuTai~updS!#)7$zH`jhY7#F}kfp+7yw`zO&?h{re9_khwyqlhtxtdqQW zGI4Ic6EE69V~V%$m-Gw?Px`+_QvTh*dqK5F7--L;-;jSRN^P((&>!5X{CghS+ioo} zIj)Bjx|f#1>-o+8cQ+?fyH!|*5{cAsivQSJ)v{NsS(F)IE0rAa$}*5vl&%Shm~>)+?4B`q zGx|U+cAQ{|6d+alFY-fY_F0eSeCH9NT?(&&51vucmtmL=L(;A|s(-m%U%8MV#~T*P zYo{8KRu1eF<3qF#h3{LAA0DPYwo8Sj5Hrq4n(}Acqt}Cu&fEPkH5RW1*j3YPEA;Zy z#D!HAdP6eF5NMG7m;z=B5vD>GFogMp5Fb;gi-};-#%w4xv9*Wq!P@5|l)f^4s>XLpcj_ z_1l;E@Ho@=PJyNRFoZrlvV;6c11LBXLxiG^|J_>cK|BI@33a_N= z+D2pBwylnB+w7Pf+qRuf$F{AG-LY-kTzkFGcko~Pbe+sqvu2H|aci{IUuQ^+4!Ufj zu%W3ds2^J?p58Y=Nr}|OERrLrz=kgGdfRk4$&7Qm1Xa#dY}36|tgw~Vq1M_p`Df`) zc+A>Nb5f1h*Cx-}oO5aG+*Pa$n{B7niFYTJ->7%Fu*o#<+#K<4TMY5bPHg^Uk7 zs@@Bt-KY!o5x&S4G@w3hB&(KAZJ9^8%#fU9TQXz|=w{u_qk=(=U~2@&&4zI$a0K%k zzBi8y!@(>k#_4Xou6>3xCV!Oey9}2a&~SP9D#Z=p=ZlnT+p55e{d^bSZM{y zAO(e1^wQl{kPPa(GBUrYj(!@you+!T#D5Qql@4Gx^Wk3L9+1$Fl{rO&o^rq%~ z8gCESJE_!E$r-VNOm<*pGh?2aG;-B*bVmJYg0Jg(+ZJIh;3WHE!4cl4Xg|A_RyZON zh%*m=mi*Q#%J^5W<+rMdl;w?TUQ_PSk*z%9L6k5$M#>sa3i zUzCh{uWA)c_liJO_9&{Rw!Xu=iWAnpWJ4)ad(C;>e(Fq*-?2YRKhFtp50n64Qjh7V zv~AA*gE`J$phCe#N9p$u4o%QHGIpK2 zlPjL@>5P_kuoowmHVy8vtRf{iv!muHlr>)TT_pkC57Tvl?O_!uoF|z^_K9*7Hr~w~ zay7+Y@wpTCLO(^?Opx*mUvE+lip)M=cFU)MQ>)AL&!LCyke~dhBQk=EDz_^383=9h zwb)WWOWfQd;=!CqPKeGS86suZX4wk=H3>p8cvdmzttDW9^4)g2=6k2Si*o(!w@^iw zK~@93CNEYSio5fCT)#JLEc^I>7#!{fMzPTiYRRg^axptcpVVXU6u30%=_fVLBNTOy zf7{OZ07ViN1Q3CPxdb9`tz?NbA%ZT4a)lg6u}mmBx|B34nTizZvEp7S>f+HIO3YXY z;~AwEnWC*-@sjN!`z{9GMpV$!r~n~BnPULAfc<|MnR=Fch%+tM#uyhpRIaEE+S&|_ zK$JY)@0|SNJq$`jD%v$ns@@T`2!K`(W+xEYW+R|@1>7Fmt=W!B~hD zHeIO^bbR^SvDS3pA|lU0m`m{Nwh$OQrpO9Od|>FOtfUe^Y;A@d`7bHPnTylk3{aDv zo*^DD>~3ZMfg9u7fo_q9K<1sFc)c(*s9=yCd-!hUgx{{?ApnpHAOAO|N(zfSr$ZOF zbKgA?erzLC>c}dEj0$w=yZx;riA9aO{rj!#Q%Hl?lS{7f=A}E_c5&DEj?Oo#$8nH9 zk;&)H@`i?(@#Nr%LC}X@0BQZLGF?gf6t-08>VJS#Ov9Ufa? zt#UVoVq=vWXc@My@5w50(3qw1{|}JB$Wmyv=2xtR*%LBGZlFA{vbT86R>q*445;za zm;;471l;Y?cmcmPeHSNyw(VglRGzcu0eM~BJV5Th<%w6Y3Jq1(9L>UYJnI&OVyJmr z^B_89sELCz1(_QrO!vJRGOs^6eakQfGv~thTO~xK7ZSs$jUqo-Jcxp>6J|qt(GitD z4c1g-#dPrtY^}L^lF7K@et)?&V?Qjp)yu*Eah+_jWSU6>Im)(+s{S|u!jLAvF(znF z^>HfChh+B2!%|?~V|{cw`e&;D+~30~?e#5onH6Wd7B4$$epn54eFe3jCMy$#x4gbf ztkwR$=czT1Lib}rhQ#oVbFUUfGzzt!F!jqZ)^@=V1d=Mk1VB z0CVUkhJ{RY+~!JM4xwgP-(*?sH*(rh%OYp~>8ItYQV!rn570NwzjPTe94oAM=+Zbq z&4qZ3^0TLBu;_?BVAEiTm+2{LuJ+WpkL<#Ncc=6U#E^21s?3wfx_|KI#>$*JxAW5| z!yyS@B(V`_DcPo22I2R#dlJK~X5anV8RtU=m_;YfBkJhz%mwRD^-Jcd$#> zk17O))>X-*=D0YH#7C1HeR7qeUO&;sDr3~^67@q@^BNn+=h3H&cg3$NHZehb-{Xm0b# zXTVIG)MkC%7KhJ&fq66>pBv|;O!Tt#%yN_{XTwyjW;OkkY))=u5G>H674`a!nuy-H z8=k)P&NZhU*NPvPreyvFAX14ZJcS&>>{2`yM#i)QhkFDR?dhljgamkPZvY!(re^?V z_&+sMu2R@pMBB)`AM^_xxHcUW*Iwd`xOOPH0(j>bJ-Iw-dT0LXr(sUla6+k?V`{vfU-2;`w0Hc6hG9JJWJ`iy zPU@u8GITzo4K~W5#g$oM1@EfGs9Iq=+StdpO^+Ro?-)fIV!(v+%Gd~zbd+}4x!k*`V2}4Z3G>v`#u*XcIP^{K%TRsean_k@ccB-18|Ay z!@nnh!XQ2Y{Nnongm9p`t}Tjs@LO-*G@xI-8pZYy>J&kNsgT#2=7IbR4kJ(Is8WSC zC~=FLJ4clxjKGiL-5>5vaI@>govC{DNPJHKj8GA~9R0+USCh-lz`xXu-_No)hdi4W z-GCc{=4;&+rX1w8bZ`iPH!;b?^is_h7dJpU>{cj#dkV8)D4XgL6 zh$@!V4GyhC-s-vYLK^%Hsbs%o`w)}lbVvN#UJRR^OMNAmrMSo zf((m%Ul%?VTf3&|C9COD$}WKqzd^$46|-aw749HG`CPMzK@_wcaDloD@{yzsQ0Akj z2!YwTCB8i))Z~vS9~}9hV3Aef5LO?FXJ@VKFoZKaJ>bBphQoPF1(0X{j6cgQqg>2I zaiK)K%0m$vfvx9DYXFUHz{N!Y-X;L~LyCI*wXZ4V`4h-lYrX)6_jOaU*8p6uf=#qp zb@h3Jf6I;L1zJ(fyB}xI{LJ~B)1(wHqIUMjYUX?Oz&~Cd)zbh>W=*U?i#7RR7~_C` z7Fs|;Lc%}YH=tBfi>29r{S0VpGsR6*F3z2T4i~;cRO-}?! zg)A&^0%jz08Q$^|cS=fHY0GK%Lr5YA9!0RJ&h(-?YWsP`Jj%wxy7fk4_#43mcc61{ z%stGcF9tmRo+ZWG_ZZ4ky|SOW@~v5FjL1eC915?;r2jjw=EvpYOND4zH)22f!wWhi z$3ar1!-}tQ9xH4)J0AU2j^%ty;QO_{G@TYH#kqg$MfRzHx{|@XYp!JBa`APU#Cj?} zwBy9-{wikK=V5;xH;HJn{k;EcS0@B;Q5UhN(LVP%+7>498Nki0a1Z#_!3T6DmC1bf zL1H=3Eb99Z&E?Ce8j+n?>m1{q)G+cH3x}oKmdI{!^Aa`=A${C%x-hiCuk)5H(Im*`=wX2X%n zXZ%Y?SYy#~eKy^v?cS+T>dnT#MKhoU1S(#AhkDoTlDlIFOxC+~{gYucC~Z4#2w`N` z?FL@cGB?PQpZ^3;mje^4v!zLL6TZWuYRaN8^xfXcmQ(&u`@ZJg6Fn&Vq-6P;jN)+k zs38T{{YSIzp=^Ahcvva(PoR6tj$7HMHb7*A_w3@k_lLL!uX+uA_Jz1~kVR525_^n< z)w-?S3T8)a+~-!6W{!`qo8p32RjHqm68L-G;f3uL0Q&4JpbGHo_|1JdoMKYb(QXNe z587NVNEFc5uQLxgs6GN@dmSo{n?Vm-uE#?F;fG1(fCUne*kCXI&+-hv zR+Yj;OodZhm#M z8ES6tR_6hRrMZLur{D)8W@dT;fr-p^LbQn1k_%8dCyzfIvp@{7(gMR6ymi7AU#Yd8 zx=(@NYWY^6JFQ0i36x}Bo^pGAF9_LR0a>Zb4`ROot>fEKchk6nZQ4kEnKvwAY#lwAkP1AI5av$BF|misjX#+prR1bVmAQlcOZ#?x%| z&}=tHvLP%l!tNZF-0{xhB-el9kBoNy?ww*V z!&^{wt__uE&COUniQE=bngn}jQm&-DIME%DH6^=Yd5;TEoVuAW&xBjcj72Su^r!F` z1Hk%NK_K0b3KYWwl*^igVsD~298bB;_8!ET=xRB&(yy)Ig3Y-OgRmtJmO?OVGQLhr zFV3MRmRr}mf3<+c&2qRYXCF>f1nJD~e?GD%Gp?)vrr!MTU>$?LJ$yOjhXKeNp9$Gs zM5BPd@Q?2lZixj?OQOsgK{{J`GYOmE&vAh(7Sly!=;ffBzqw)^P(gm-e;_4RJh)%^ z3OySMKnnscW(q4!Vx9w=Ed!L~1lMd@vPuTfFH@7V%RWXJHYxvpT2tto7!RXZa6ZD> zQNcFfeTY=u%!!Vd|9IQPH!*jIW}UOpWR)@M$uE%D0K0e(quo#AMF5e0d6T;A0MV$zj zRFwB0WJJsspiJywAUF_#+nwU~XJQ?aM;0&-N04_s?^8v~X9Gf(#E}6KFbP+Qk7YYJ zh2$)!FCPUpIxOh5e+(pd!8=#kUQd(eDQ>-N$K8|7-9Ns#2dHzsAcB~8T zoIZJo&0(-BX3$|#VnG&hiiqCwZC@X!{|9oI+wA;DtOm?$a2pJ&K`6TK8`q?pGA6{nE8-7RqLd-hAd zV@qZ^w-`ZjE}qOoXfBaJ0m*Yd)(UU46J{ooPg`c$y$nNvcpG^n5*YX)w{K`0{O`77 zQS_2hsU`R)E!@EyIJMPDLk znNg@!Y+8!>zps+dUKlVXJgaQ7_S!sf1;p;l$4?r0T3$r znxgQ(DiB*neh##}R3VHd{Bhxjxt=jzhp%sg(cp$`KB7l&bzR>;ZQwlBE*W5_e@$in+sA$WJ;3g; zqV;=HSEQ}=`@*yLn5;=`%51g5CbC9=uIjSc7KG7w!~p&%S<|p7M>fl=4;0_Qtg8gR zuj(ioGhch$ZbQvtgW@2OWnOaQ(gZf;7rNxen4N9UdGH|XGx8e4bM3{D|99CFd?0K= zAn0Yt2{ZMoM?dL)JnC@%B+w<7@&y240&TDXA+eMcKC#*R09L4z>Xi$(Z*rB9AOe11 zz8On6>mLxusD#UT8!dpLuB22$vIX6Nx;>G#b8o0GXS8vzrC19B@>)JY(cv0n-K6k( z%|VLy`FX%Rx4tSU123?}RLBLo!@{q=9mth_eANy0OPDgb*QjRJVAL*xL+5NZap8{7d5Xx&Xs2HpnmL&!2naQ)D^RDn8kJcWjmfR2g zZCMz!N)Liee&}i*U5$9yHgQ6wzu-LQ6EE`R(JKN9qk%1$hog`9Ru#WY?N~?U_CNOG z)4G{;z52Wcw$rg?`Bt*?E;P2e=fU^peLW`$PRF5$$Bn3uBOZF*&q5_zv2}9!Xra?f zUjX!1LcUOb1Dz(yr=&nQSG`>W_M65Hpm+c2;K^zA03;k`>ivEkXqgQ7I1khf6hbK|6M(PL$B6T);hu0Sm8utP0GrZVBibQ?h?w^>Qx zrDt`2#zeEG4s&&mWOH;z7h6pErGt)UP9SHNk8mSG44N%cokTPCDiWF-?P28wlDi|O zG^<6sp1SJ)MpiZ9!#1=7_BZdRuXn8P=)dO&oG1?WN7@-c%Z#zaHXs1=0g!TFJP)v4 zbpU=c9*;*J#nd#3^XGj{qlC7wCeD)v`aV11=6AQ)o`)TMBXoG4Y;9V31`(F%7vR*6=I?-RVP*EGl(ldN-99YAlfA6N@nQO@M=Nyyn9 z;X06ylpFBIfBhg=DUM2-`7|$B`gL~VQ@(4PZqw_s4U94n&fRw0iX< z7imq~W$biAfIo5rHZ0wTLT#`fl6v%$mI#9X_jCm8 zqRCAqX$B}0AD^)yj@`BaKP!1TolT#v$Hr6p^7#Fy(YM~(>Ki9IB`sM#Hq}rU}^_-te z5E)5tt77Y6*$%-IYfc52f=&+?TWrzgL5kwF%L>`!FN%92?pNZs+wPps2_nu4|e z9Ali8R(G0?{68i&!&0nN^3)$q^c`}G+yR14IQa)!AkF^R$;g;$eA_u5tF(sps=yXI*ZmonW~FNfk9$%RVNYMK97?+M9+8ZEnXhomahF|!c^XWAC;Yonsn!k1o`-dYdq zf{F-kQ;^@^Z3l#gJ~D?|8w=7urXI8&A$m6D_&CO=q!WNX$ig`T%LXl3^_8GSR-7ea z=`C6j9Rli$U*?**q3aOr37{G}G2YT;x0>-jxMLeyTbl(X1UnXZ7qoqU{~5YYEECvW zDX&*_i#Psknh!AzxS|tsKFV!_uB5P`X%*1Sd=7xa3$VSGRPBlEtCG!ujZ~N|h3E*T zs<~ORW+nFY!_xGcGQj3mQm#wkgK!w+^mEL-GgFW}c9Xo~lta z*GF56pTgo!0;^cJqBQ1YwTMN>`;GIIFyU#9qp&Mx=6KW++#W1umRj|XHX6TbhE?`& z?KIDSf2^q2{i#|2Gab!~SrQs@X{>%VuYrRN)M@9U2))NL&QM)qGF&4Rzrb81_k-a7 z%+flF3#?&jhD8B!=X4oZ&C#8;6*?SPr+FORZV}4EfCOo|C zhfMsJN;?XEXubX)xIFM3O~?mGJg|{crIURH=3XBX1YrrNFdG?iXb!&Ar9>UXJ|8^! zkqu+UYy1Yc64r%TQU}n15RsxC&^{-`fj&`JFutnXl`cm+eHNn4F^2;c(-i&*Hx|HH zVRCdsjulV|ba7kx*68)Lmr3#Mlp#cFw7Cue+yp6n`tqVkL81^*P6hH2`*zL$;VMAw zmm86#mc1we?tCW_keh_O!!2W3-#+rMrK`N(pxuDaX(5p(uizlG$m!H^Xque_pB-Y6 zgyq2$bPxV#18y#21IF6$%8WKPyrUt(0@{F@6KO)`L42niy=dFOX**aI@f+z(3nuLD zYe{nIua!ypWpM$gDpi~n;S@)+#tB{MeHtsP^;r>}-N;;xA6wr2cR-7(&{)|R;DaFz zUKZexduA{!c;?g=|2n;R`DxIv69%ByN*{R;B6vwidCQCV z+LM_C08Mnm07{X|KoD*6FGR8JA%=MUX-($1mr8A=#km0LrhojwEA2WN<_9`6X*m>*j;LMPora<^8|OgF}O^#H$=J(PlwB)YayV zpY?CZB{*Ok7s-VejYU8cgx*+IT0e2hn|SZ*E$f-!%pd!9nOE8Zcx(#UJOSB5+abW{ zps_a_A1mEDV|CgGU*8k5k|%u9=nYp!#sljMp~XA?D!f^c&F*1cl1pE)GC)(@%>26! zo^uQ^^G^}T)f~11aW2p06JRV^f(7S89%xxxtw>r;T*luh}t6RDH>xYMvs>R(X-h zW!Pm3ZBZ^l+ceRI^k(bb8rdgHSRrWz2T`nl8(xJ#991@rM825M_(P6RU}@e6hOK1i z|5{?^YapSj`5W+OvWv|LR~nVT(rLDF!kI$Q77~vW-Uke*T=x%MG)f-{SH%&&m{?}B zCFDJ$Ky#*4g3+3=l<~Y)FeI~8=kC8fnXvWeiJY)x`%z{{@sk`l#&ZuNU8rBSSGtE$ zM2voOG^>^h#>$p}GeJ1SP52LC_Z7S7z*VGr+$1za>tS{Z4^LU8BLcY};gyoehV1Me zvHS@Z>dDwx%x|c!J&*s{1=tomVlm)EmA_N=#yKb>cMa{iGVM9f=hPV6d2v|qVD_OC$u0W@fQyu% znDU)x33RM|H$|OfQN7YaGYcGsWDt>h#DKVJZvEJRO&F+)jNr{D|8DC;exJAals|&| zlNfdrJZHOW5}`+ovwv(#J;MUg^Aid>;v}8DXy1m=Ex}O2kyZGIh(4MY^He7Y4Q`RQ z+moYyI8@LBP_5_3&ym^!3e(!6nNVB8YF$$ihmULv){vZDN@{iW0tV$6D=@`s((^Fs zvEDgeo$Lu_k@$tks1U=<({WrVFuSpG^nC-N+CcV)=`vk`^S#<=UF8^v8D#t`y~iU( zWDlQ{dz`6{ywpWNM~{Z%)jF<=sp3Kqj{Zx)PBj9jFMEJHs|s?93x-sH(GWOBUn+sQ z3&d02dsWrceGk~D7DsarF~YW*{Oz*&{6Ca<-iYh}o`OtI0=}(<&H#Mee73+)n0x;4 z&l$nr)4j6m7|*cK%r0Jb{~7l_E6)ZODHn3SQ1&5AkX;4;CAJ>p@F6+Enlv`xc1tCv zl>7|@UYD_AR^nSn=J?QE#!3dl&m9xn@Ds>XZWvhq)r%pXwdlS!0RZkrUxfZ$S+tr} zVO-&pkn~6ZqM~BF?pFXgGG%DUit63!#s^0~kx9h=kuT;ELXY1+hX8&`zt|mczOljT z`dL1wGjcsil}cK-(w-~s%xVC*_s7~6WmcteqHlSzz|mm{&@}*{FxYYz%{jh&283F8 zn%6UvlsFc9HD$sNB!`b%qPpz`!52eyX7drl&6V0DArPV&HW z*T2JD8x%vkaiPE}lnS7GeDOXAm=k-e^J2{=iVm1ka=#>=w7#Y;H!^AUPn6{~*{ zb3nLx_CLMzs(%>-^t~W_5~|8$A$0pZq+dOO$$kqX?OYgR_ls%7Cks92f;PA!m1zEb z_Wv4?zfc>!-6q5xIpGuH_`6r-MKB7A9Wnv z&oiANeU}10&j^2F)_n|~jBdwmt~bL#r5;~0fYxA6Uj9EOf_kwfsH3``5r4>Bxg$h( zgv=WGB9<#d;3fnE5rlUOL0}C=bm&6m+)~j&1QPfIA;r|$6d!p=5I`;axw9@F5N&T` z*LJ{Y+ZKAsdx70M8Fp;Oa;ZrjeC_O?O>#%uJ$q*YLSCZaZ?aR~MUAAPw)rsTE<1H9 zahfUz<53Oj24t$+TxDUZ-2}*kpZ%#k&@w==XT_s^@_~h3I8*I*y&-pit|kz?*iGLq zfZunK+ zhVF`J)Y>mOKkn^U12K#$t9HHq=c32saneu#wDk;D?9^}-<4J>dJ|ZDLl=^1=c`$aK zkGLqHAz6LxNsHlVmGz#>qjO4HL;*TQ*BD89|It!qB#S4JBGn&4wawRi)2G)Po(q)F zjng@M&4vx?5V6k=VKjnOpy^GLjU`2Y~b|oe~~OP_*`NaK4gUeN4O!`O2^?d--5wR-kY;M%QweQjeRR zlnz2u1u`m4SlSTD*zJVyK>JpVR(Fx#_L#<}Hz5Cgz`|^=V;-d>*4U;ss=EP&g#iNm zcvmESp3_^!g=u|0iBn>$yFD4sby=CNov2p~>sr7ujNcX6ueLlY2wEI9kcj*zYJ2kU z;yrq@y{fhohwZ+r>8;4U9Yxwd!o<%*`n-wHWWQQRQHs}LHA!6QGvZq+?J@VH#LCi< zvjX+Z1uiB3+CvtshoUJvCs2Z`npUM0!^$8o!Y=sWurxI%I$t8q^}v$S%hFyJUAR=JUUenYs@DMMqgb&zME>?bEjo;{1vjwVLJ#I$fq(cF1hL`kO+AOQPf5G&s;!xpwtk zT2nF53YH)ooi=3Lb#m^#=ZCWtoZIvs!_fa+U43zy7Yl{&Hiw1{RcA-6)J(YyK~YnD z&A6na7K7FgJ>Wk>Uy>PU>ahcBs|%ilsYJr~xl&3C>(+^>m)jj5qigY*K=BWTVf^Qx|JK>kG#{5eZIt0OrlOTiF{GDOv1(UhFD6e` zLm%7KxwPDm{}uN1%71&fx=~Z9(xGaOSw@`_<>?@e^AV`5)ssca4ATZKyAcp5m)5TM zmZN;w6-lOrfQP}`sH|K*8^NHks3|5Es`V@1jL7~l!n?kLJW^AqIv;t~k{g-7!dcYu zO5{Yb2v1jtrKyq)WVlz2PA7cX&X02g2~F9Tx$>e)h}C45NQ@Mn)(Jb_3L1=9;8Y=p zBJ7p4(cmx3taxG;$QM{Q|xL)|1Q9 zTqY?cIIAUEiA*bOq2cKm9Z6GQod%Mbr}lC-RUkuIt92?43-HFV5k8-hZgDANDprHa zb7f?+#;q-48u({Dqr4Y#NuROhF*lJtrtj`%bB@G!jOl1}ll!l#zN@|&Ju6=L+Nhvz zH0|BhsV)CNw>iH==2!7zC4JVoWA$zcfgrPAP7UM?4je)vXQK$awx?`<|TZQa*AG3^&GzQmyfomvk?=YAJWu?X=AMm{A=Re z_Cd~2wkrR?@#+eHhw9i64m1d-mFDz40_JB9vX$S-gJV2acGrUh)!fr@i<$0|@ZVo! zGhhwhcg(7(C;~TuaWA<4Tve%CX)DTQe>tS-l1lV8#K=wlO%>(B;DTcf3DJZQK;HBh z{tTMsJBoI-m2_s5_rBLZ>QS~>lmu|eh10#!+<+yQhE$V7n3u~CV{YpXsF_5`C}o`U zM9Owt#RvMk*_J>gfiM?)CRI7r;qPNpFSj=qT8ZzyWr+0Ep?plP87XQB2f9372oz#; z(X^3!V^*p$H_-SVl8lfT;B6Sa6P!cB7FmbAsD4nqh8mgma?dEwT-W3C($R`7Gy18s-94$G&*nxd|XvXklybgW3 zPyf-I0jox)pJwsOg-MF@o);)Ud}YR7%3`$8YQi@H^tuup zO+_?86ts<$E8fA@r zT_P(=Bw+)uWOQ6+@gOT2`1tEU2YT?lw3Jt|%bQ0eN-Zs|<*)V?%?PKnTW&(ATE)&GY?~gjpjcxi2}pELw)KxQ*3ae+T8Xiso7!79B!LGj zof3EpPm$l=Lh#5*ogKq9GZfuaCC0kM`f~Iq!NNg-QYu&Uozu)bh-P5dI1e|tL?xsO z3=F;$5K7sOm;ew;Bf$$H6008FVC*fy#1uB$-u0P9kx+K@qN5r3ksB+|jN8*B-zas4 z6tBo|CirOxMHA_fQG7xC7MdOM2{J=Q^gf~kbKNQ4mH>=oEPIWr6QvWEx`6M5U#8I=_+bmf zY}YkgESoE(b8Q&#=8fN!Wd4f}6TT%*m%`1% zvkvkn6{EXos>lr}lZ2_bT@ue+L|Amws@qalBQXEdlsT>ULhg#@knzC%r%U|^D zwOO#7Iumh^pA`zp1gxwi@)8Rw7|KJ5GfRr}&1>UFwF!}<=`~K>>X(NlyJBtpx1rBC zcSKfZeb3y9i_EOhfRE2FW@Ba-*hRL&Awtq!vc^w(+q1#g$SoQZhUyZp`5jhoVRhZE zwEOaOj>o9JKPiyxOJqs7gal2^9?|^@cHCi&<*g4O;rGJs8u=lJRvY!qlCp5P1)|)W z;JPuhAKu^;&A}l%+tye(qy$emZ^7P(^5)1v7*TgwQP%2MM2sufhOD@yMf|Nue?9vt zV=&BotV*@AZ;qm2!_YOmId1dKWL3Xf`6#VVL2cj_+hb#q!^7epP0~3Dt&cl?yfrz* z$r$%Rv9`B%Fz`NNFPMy|A=d@5qIboED5})O11!@4H+95~T1XuX)L zW8iu`(LCn$%v+&h6Zz5lhP3#yAT3bGGD^g^s@9(SD_7sk2c&&%)FFU~!yR>e`ZvI> zm80`-;0!gJhyS`owycxwZN3&2HMt+-#zFh_Izc!(s4!9MAJF%!n-2&cIoh=bCh!T= zz$O$#(3(){2gUlF56#@Hf2;?*zx8Oe^fCqz4sNc=3R6hD_Ap(!?d4I1U2)0|SO+=gN6rqSw}A&2Eb!!F?@s zsi3>|Qcp8U3HO_<v6%iqD_Mt5?E0&+3ehtZ&<+7UHk)f6OtJXjCylH^Cr1Ya?5 zeAz>gBL|!kJ`FUovhLbRq_>>4|0Gh#oJAeDSG-IT9gV_;yM-o1~tvXP~!F)*hY~2_w`24tO zLUvHq$y?7FT=~aIJjTa|D_C7{E1cMY91Zox+pcsuCw37Huoej|>uZ4N!kN|BpMSvHRQJVikMK(qb!*4;E)CXQn&%^Y1> z;-z>;#cNb>D$dz-uQ`86Hc0}8X}&$4NHWJG<@0g42|fK1r-V4sNU1%NP|BDQ;nKfs zRBzh)r5rM}A&IBDQ)EnTfwPU=5#OTk8xo~&Dt0JwOY&|g+>3Er+qTzH)AG8S>K&pj zf$dcF4Wg8?BjvhYJ#c&KQ(ig0pVrlU7UwaSYCG;%N10{Hl~s-asa(~EOUaYKzQYTO zj4}QFw@#b7m1)+ypp_5tI#;q&XB7OnF0)h zrH&kd{Yd0pcA#j4DRhGvW}u&GcZGvKM}IFyoSjhrkHw#u4+t8ynYQcb6arC8h_k{* z0tJ5-{2oD$Z;?gghHV{g^!p<52b!s;<-FECgiaBCr1C%Vc7ed!%lik&M=eKW6H9r5 zY#aV5_$B-@Tn9fFo|@T4uUUTaYlQ`ba0jTVKo~$Nb*g{d`ljjc(_!_Cj*C-B3K5*e zS?iW^+Y!X(i;v}?VSvMp&7N(auYe)lZxKa*9CG`4i++;ZbPM>R>{D|#SsgUj7ItYZ zs2F$Z0w}q?5KV1t`9>wC<_oxw9I>q+u`9@2jnWI{7Z#VCZ8O#z%0E6`kpO#R`;HLNOkuF%R^&974B}f0t<2PH{G6EJ`$PU74@FY@^;p>H>ad zw{wFIZr>SshiqYehY(_aPw)2WWjbiR)&;$^h5~eH9Q0nff6!Ej zUTKdv5oHZ19{y}Vc^EO`=bcXzEI%0LSI%XTgyi1XSnx1>4VCgg*6dM= zAII%}j(THP?6my)_P>NNDZdk}EXw`!UBWe1ukhz7oUf)i(*CM(ejeq!E9^}(E#2*Q zJ9)m5XhKg@7fISnEJP{Gw;H-8C$4Mrghz4j&-?h;MUjfzRwDwyQ9) zC~msOxq}Y%y@DZpH(<%3NmfP=ye^7hxz#;}Q9t9ke%c~}V<#kBXhOF?mG2VVGiHms z3(i&+%`y`^QB3`p-u7J5yS&3@|AT5uJ@$=}q`CaJ=<9EExUzEJmH;LcgOM1ig*1c; zjWqV`^^FvP;oLH2Me_NyA#tXy-zr*j_jG#<3jr4p4873|^FgUDhYvkHc`%5~qRJIo zl?gdAz1;E1vEh;Fi|}Jaf^K;psAHQsoALHLE2yXy zST;`x@9GuvZ`rW;3jgnvb%_~*-~75tToLu!;5(#B+*woESw`vr9DLL#K#Zy zq^9j%kVoWw+TF#rJ?qln&cBOFFI!)Na3XUw)qeK}w6W~e{%(ZY-cVIr1}<$Qi#~R9 zS?(aGP?7M_x|I5ZWvBDED)5B=IFhe|uTwn_Zk#Wd?{vuQL_h$;?%*9K){E_WPtQjd z|9xf7BdwCTy0a-^RYk|v{;)PTeplt)0nTCyKZFDs=;t>`ljhC}OhJ_7pdr-r(x-2| zS3ebF6BZ;$%1}9T>1QbEbY;v4X*9Wdao5x0-EvA%WPiQEI^I}HTLwOWo%8>DcC~E4 zfR*6s*O}~-(@tw`Gt{x(O4|8>VLAPo)zpFRTxJ<{kBIp&iM2xq@pF6vJ0eqvVZdlh zWC~$hJ4Xnxtn?Z%njm(!l}J>F-M6+x{@+HYW2q3o{IW&;-}B5^2rLDA2sIvVmYP6>>;e53i1Xu0W1-PFc@DEpXkhZ%$j3qyTb%9nZTi936xdnK)G8BRL zDOHgd@9wod-4)M5u1qA}74Nd$b^VSb6mjprTEu&0h&S-0yj|+Lz9;W*yW8t`w&*Dc z%-k=HwHD0-N9N1)k;7cT`0>+B1-a8@Ds&7O(mn&zVcG*bv>i<5XVBHlsbnp?PKF!FH&FdU->(HG8fQ;eV5%bKJ^DwSN~ThtB9+CCH66-1$bfpvk?0m^T_81 zy?$T$!2RsfB24%HgPvaN@Xa}QW(GC@9}Oy@HpOia{xT}%feoJ4O)M|K7EW|KD{${h z@=@bm7J8EgMM_=;E)GEUap>I~iF=Owx6=uSz`lb&E~ct4{_EtmM%9kmDemet?a$%V zDiho#l#KYaq!PG-Op?IUS3vFV=C%iAS-o2k*EI$$Y^5y2Q!o?}kZBxj!mE+nf11pywF(-?mpe2QoZrYKrVY0#7{GTCF98EN z)N=c2p&Igt@yQqL;U-^NlRvehA%=jY2w88nn9WP)1*0TV!X`IsEz(DEw{M8~=)w3U zuUQ{WA)w8W+^iB=w@&?J0ou!3%XM+46e&u4=kfX(pJ5DpV=3S-YprJ5d zT**Cdxh@N9ramcnID5036w*CH4((NS1~@o>vk=)v``X4B)2p1!hVZI7eJvH<-uK(# zAJgl;p(J6ZEVtSlxk+0nRekwsU-mT*ON z?eOhV-Mti@YX9@HC{-IeYYj`>%Zk8yc{ftlOEo~-Ak>a#kp2D|Miixf+ImOO_-;*%sl>>7h4x+I+`5Az}v!qRH?h`h-UobzKl7MJ4s;G*ar z`ObF_S*tH~Cc1v$j510=yuN{5Qpr5EYrdAkq{Vcn8aqiK)2!a0t+USV)<#eHYK+xB z11tT(VBT@pT&~3HL91rI9Yd2m4m}=M*P@xnbTr`0Z0VoH`}QG_@Mz|Ip$@v>hqL=W z_Qf;}<`WpYyTIc_ygM1?xKPsVWn!+Q)h?LMp~9_*B+`ZUTl z+n3VOkq%oC?nmVm$rS~Aq|FASs*QDXW`z0w2ev>-zZx(eJ@OnMokyOEY(@QlTs_|` z5_^29$d>E!CXv`9&gQj5-#3rM)|V5jt&&Z&MY4$tG)5s4Nrw?2ePH;?0IeAak&t89 zA4rcG=gb{rNS2g#LUR_pbcJKA2ANGx-Wzr(}|=B8=_EVfTavO zvVC@brjshC3~yI_l03xfx{vS07%q|#T&1M3$}-VD%9|=F8D&W^ zffBw8V1VV40ES8>L~i6D;6RQJpoiIA$q>2l)<(&hMtK(COYyn-C^tsY9}><@4i#J5 zKAa1j9F>BT!$)4_ln?DugOkzE71xD#EYbE|5a$h=8;uT~;0Da;IHy)Ejl6>Jx`(jM0sFS7+Z9c1lbE zje5$sF7=Wlc#V|l7yl`2q~h5jq3k`0e0Cm^01m{Bl=4u8eY9Ki3eJNGg0oo%Z<+LG zi~DV;s6_y$s*hZ!DKm8v3d)EK1%2c)_~4Pt;MMg1arJz&GI)u8Z%PI))#Xjf;8k>d zQ!;q*zHeRzFW&d5GFU}{wewM6AsUgf5=FfTd#r|ipu`YJdrJw)regR(h&+raAVLUb zA5_-YB%}Y%RLM4qC<8rFQaAnv>6c1GioCx1{VL8vKs-0wlM%ez(T2w!0wd^0(DUwq z<00rH7eoPoeU3I<6nG>Skv_WNBjk9)1O_eTDBmt}l;?t%mb#Yp@+0> zreCV7ufKymVmON%QoR)=G-Jat^g;w(nR=sWOk)fS zsUtC+AP&8fP!A7=EOuj*s~WH&WSZ&#Gtpd{_Nm!b4sES`sVfd7X+k`XPz7KUUE};7 z63;_nO;@<$-;~{sqeI-zIFG{=;iSYvBBcF-L6T@3n z^^k&bv)*b+kA!Bu)dnG=0dg1~Bk!&rJ4MQ6&e_)SBFa#xy-@W?MKgb|mX1uA@;RT1 zWT39F=7XOS)*nqsHsiNWQj@5BLR$F(Brc{hyyl#^ zm&l1i%>baZxl#=quXekGkR=y01ioPm63Cc6wn!>U8r)H1B;!nR({hA(xWjYw@h8Y;hxqAXaq*1 zAeJk-vz*wzqGc6O>^n#u;VBCd9Px)oP`;y{)c$4e2G95HM;?;4n*Kkoo^SSizeK+` zb-rJ!%bPskucG6dI^Qqe_syU07w`MD^LMi>+;Y8Fs(h+c z`9l~4$d6BTDidM=smdgzOavi|&zrpW{`U;HIOOww7q`e?;9pN5HX9$AUvWc4qE4zCw#t662;)*if0h zTEgsg&Vii|M<~re{E91B;VosDd{1{i>3pOdHZtN656B2TO#CEf5c8+&nCyFyA!@LKo`9vVK9Wg6m%riduv1yfDt2fD8f)C_goW#qRn+ZnRBIs zZ#82XE%g*}0=#%5N;Zf-bQlSdTL)*JX9~IGp?J^YcdyZ%xkS?$eUq)w!y~?C_@7*u zAEN|CMJNyBK{S;Dyd@OiD$OrwH4^)H(P>vi%s1*CMS(`W)2iRrWg-`s zAS!}~_^gf~!;V;^F(ar1f+BuMmANB6fJ!G6i(g7oVeI;y*bW{qeq|6B>iW|4qu}7G z0pR-6`)_BDx4o~^j=o#e)AGaaR&{dn@pc=#T7ID2p5D%Kgx#LrPCK+VmAS1c%xzN- zCIUSN&_Tbb_))D}76P!oqrAg7Kqxw!6ue%Lj=rg^N6lkXzBP|$l5Ynmv3zShOZnD# zbnK@dKD2yWqNfih-yXrHPhB=uA2vkggA+0!L~d_7Q;LK%^4XYC)-2RBcrOMMzr6KJ`2@00E(_ zEG;E}V87L9HTN3_`_1-#zH7B{tyOEtER@+?&)OYEmvwL<9rg$eU1bzhZZgPCd)PG( zxcoyf^e_rjM>^%C)9j>3*O^TWAI4(%=vj*4^=A#*&K_@jUmscwuO%1IR(0}W#qfG_ z=4?-IrwTUZxzapH?al|Y?arNecRqZUyYu=p>TG9^x4o|qy*sZZPtI0#@?m%9_2!e= zp59K~os|<)vz^+V+q3P?2l4LQdDz`~J2vO_Cz{!w9zQsTt~3iyX3;C>tY#~<=v%Wb z`gXkN51!kiSFxKlKK7NZBUP(QIr0jV?+6b<5sb%395h3G?+%D^663Z|V0e&a^Z?5H zsM$@!e&-z%qeo@d6n@ z>?2pMgK?wYmPO{{Ri|76nkj#|=4}3Q?ZgJuDqM7p{GvM-c+Re~WSzBEfQ-`uLQTtw zIi)AsWp@}?r%NqBbc?ABKTKxpQiL_Dt>MirL!l47t8_d?+2fbU2@$(e3RJ1k1s-A) zaFS6#1LWZTolL_Zo)zxrVrx`?Un3FI8Br8Rg%~SI#$1DxLpkqILPV`q7N6`#o@hmW zT_|d07(~JHh7`agc2frPQX-#+fT2;~$s9(I0VhNvt-=hwGuckL@f;iO)qFk2ZTBPJ zkufAa#Aik1ExV2T=;cJKi`6 zU0H(A*kDF=@FNZ>tD0>Ke_i3`A9slK`2dC2RKiWET1uZT-n@HrcKN3J;q3jJFLP8- z0=1g{%2B;jZ^ZI9HQwX(fBlpA#uxeh_t?iH=%q?y8MM{!$Ym_~LGYh+3Bwdzwdx$s znOv+i`$xtSftrzw6exO~RzX4+z8rjVqXU7c+*<$pB@kap?f*(cby)J1#J650>L{*g zSxp58#Gk!N*Uzf;N$3c_wq$jMs3uuRi$V1{5fQ}{|8jXCLIj`>P#6-Gd4V`i#N~mD z9kF{$j!d@$9O(1cdkBn(i~AV4=}jK{W9VV`RP0D~U2W>dH|rvX=2AUy z{~j`DC=+~~j$rVKGQL57k^kIRirw-nr%8w^LaWONi0YC}ObZp}4@9Af2Suknd+IkBRWeN4`TMpP>-BRRGm24KZaTyt~py zH{oyJLvhu67cmr$u#Z%l7rl6S>o+94=52|PG}&B8Jg*0x>npuQsOM7;8P{VP1dv)-U2umXe6Y=KW^N`BcLM8NZ5rf!M22A8LdYa#h=#bpR z5rH#rKyrTnDN+aCDSKq3$!RosB@-X%t>$|QKIvRI4o39L?sw*&dtzrY#FrY4+9)od z?+j50-p4p3J1HMi2_nQw6uqH(gy=RPd?EGj3R?S~#^0u?u*lP$ov+{=hH%6eANUGh zBkF`$CMDv3$s7t_cfJA<(*@MOwe(*yFpBeE;#>Qb@6TiGa)~a0hqT=eN#wclvN!TP z#eI@b-|$83!;#2=;PH*0k|=bLj&qVjHTL?iSCvo>yq3VZ%2nL6jO*`L@QYc6#@Nr| zhrw5HRdFC=9x|G&(L2h}sG`Huj0`9o`dknX(YoAm`Bz@ay;oM9eiV;`Nxy>Mhe#AlGENSk@^4Hlk(jVD=S~$vyw5^} zWNk!sA>Z(xAgw}sDkstO{u{tI3OzwI@ax4p5D@H3=_onNm@#8nsV|Cpls=zjRpftGUMuatA)qj&I za&ipvvTWU}R5#{}@jH3`iz6V#nJ#cKHGt-oC2+cpoNvae=6J|ru(YSRc z8x%5G3c+7*#XXR_sS~|WJw*s4H9{4Gr}Eip zjfa45VT7v2^R(av8P_MYV=ktS$e?vj+|;AgfjQ|Ek1h-TwT05d;R`RedNll2qQj0*p~@)#3{=MHzI$W=jo9t%%+iv8lRZ8 zhz_M%NhLg7p=1rO`KezeVtcQmsg3Pa2^ko7Lhq__9CC0mv?xpLroG0n4=(=kCdt)i z&bRE$rB2YcWG9kDZ?k_}(0Ix324bFO|K?kNz|RdI2H3w{{L@&rQuBE;bA0^`bKJ$h zikRch@v+wW$J#3W{&kzGGJuQEZ=2RKl?!*FBaUNqQo_b;>cMJ~!7;U3_V^&K+q8TP zQ4gKDqle{J_0egaS}Whq*oU?7?~LDGEhkOhLuZK5Si0+00cL64qxy`11jI)PNK7er4Q;LO0tf<# zg+AJ}zJP!O330Ub!Ov%DZEO|X46!Fq^0G9lk;2r-$g7fa%ATt~00i;z`3|^|+EbFb zsS1>w4;eMGT&9w}pmf?wl~W{@+)qxOIFWs5d0! zI$2}#^bEzMWXLN^(u#c32~E>xv{bK?Gdlm0nB~+AX8NzY5)ij8RY{!5MINXkh&kl# z68`Zltu(aEf03{4cq{nC@4LjI337Hk61nw(m@R>y6HRFiahZ&yqHaiTYK-hF`qcFD z`wqHpS9hi>I~@vwUSOL z3gnjavs!^t8LKsf$~N}1B*_^_dM5aN6fyZn!Xc(XFE%9z8)ZCpqLk25u`G_lQZ*rO za<2RM?owng#hgtk#cDs``*IZM^m7gh8SZ08n(zp4i4sa8Tbd6>gfk(Rj8(BSdHov2 z10T8OoBR{VC(&RiH6|3XD3CfsObw2Y$0EvI3trUJ!-8R*&l`Ccdmz-S3N~LLwGtx)w)gp+)AMyr%V? zon{@J@g<=Z;6QF0AsJ(t(Ho|z;Hy<3WPfr@mji)16D%JxWMVqRa*f-QW=?@*WJ2^X zH9g88gi_zsW({GzR#qs{OLU7Ft?#rX?Gi2r#N;cS+VIuFfx@jqVz{`bdQ~kr!EhcC zIrI2nlD^b;+I8?-dU3y>xVY1^nO65zV_wD~=tJ!B=^GJO=00LXDQuMR5i?E|v&vM4U@9}8$;lWAIMDR0c4z4#DnvZTM;V=iGl>4 zr+5YG{ovJEvL4l!Ij>55s9zK;Mg^5+XWV4%mw%a)pHNOjHJgD*d=A9)Q8H^&a;r6D z9o^Y^YZ@-h&5keUPk$gLOh@~CvFg)Vp%$y~$vqC=p}Ac#1Ss($o%m2gd$nn@c-e?0 z!w_Z21R5%De2g7ayDH~AlfP4&!x+mACq7{sf?No_nS?oSA$R7L%56?~S-XnU_#GZ$ zriuPTl5#)ULSoG&Kd(rq5$tmS8W3p=4UrT-e0RpsGwlDdP|&M^5g@-H5S^h4Ru9xc z(gjX*Os{#{Bw{xR4~7Y+_mGeJSQNY%!m-kN3NGQt5>pxou_d*jNkVdoZeF7?@`xxU zVPF!8bU3H!*vHp#?4wFAF1MKc?X0(nYAQtF4~}yoa7b_`cYseb`cO?Xzf5{}Ps%~T z*2-{K1@z8$kPCJtGUps7XMayDG@%_8p!U-cI@e!z`TzGopLi|`(}$ASsSQB?F3!rz z0jWM$_HjrV_{2dq8={fMQzXO#>O%ce6#j|$e63#G!!Zm5b(?*v&J_RZ` zgjN+06T50+zqtY=%xA(Co}9kO@fmm=@aftQfikf7VW`DN!3@{M+9*DB^6kzrAMIyM zN9LN4CdTxd^0Q)L;tIIRyv`T*saR^6&n~mf47Pl2er=!Q-P$0+E~5JbDb#9|!H^5j zk^7wrjjv_GcjT%~D74A088Ky|TUrC9m0auZoI1EZnLobE{hUbgG3? zDUbLHkB>D~Rl6xf(lJiESjW&>`$sW2P#jUHy^M=^UGjT)#F_6-x%X&k5gSGoN}b3O z9AhYl;#h5~yir}qU~Fm+-C6)cb|<`>g#(>n#xoxvNbkT66gz@T__I{;^{*1iQZ_3q5C~eEzNaqQh&`ws(`(DJ6@PUX)qr$JMh&v?g67T#A zg?x-RK3_{aJC~v&%T=K-*%hA&d8Er8%Ei7}{=GQIGgPrILTjO@uRpGB2TI%T)^~m; zH)yOCuf%huD^(DpKE6$ILH7{2MnRGi{|5RjE{Z1Kkens$`jX@L`B0w|#clw4h~Jc8 zPIqa~IEN6^ze;+5r7aT$K6~07?UY+7 z0xw<^bm+wk)4QT*J@F=AyeOo|ix+x9<)>E8qtt#0iev>cb^yV6|NoA^&(!C6PyZX#0`UEPg$Br}oZeT0%tIGQd`cE^tDIJ32P)sbeXl<% zzb;qm>5CWOBLV0ZIg#@ChCMQlD@0dyjmA)#KUBnpwj~WbqnNHJLy2kDT1pc7J z2eshk_n+=7B&eK%iX=p(3Mv6)Lw;qScY0rUy7KMI_hpOY0XUqIH8A;^sIDsu)e9+8 zm5N?Ta*me~9ZnEjDVGdk(Sp|q(Ab!+W?yd+q(7mp5udsE;5|2ioY?296lT=)?m3AJh5FjRqe*a2`vB3T%hG7`c;Y)M0|DjCX#Lf;CpuWiU9 z;v%Xl=BPtPHK{7(#x1Fj=t7WK#YL3tpAu<_K*oYGpY8zB-lI<(tNa+VSLZ@I5 zw8S0R#rM32r{GiN0zvop5Hs|P@1V*T@ks2YFWZW}Xw{!_>!+a8_z5Sm7J7@*L5ndt zi|0s;mA4SnL!1y&J-RYUTHo2p@2g6s?+lT1omxs~|KgJ-b}A1IpmUd6gWpqd{+pC7 zX}1o!kmF-%;+r6jt7rexr&JEiiH~A?n{c9mLUjkA;VXOgFAlYo7xQmbGss8UR{9W& zqR)ImrsW~=Q56h1XG)ACS0)FD=fBmpCOLh200;}@#X-KD1J$T9JCp0(!9l2iqgo}T*8H0y@wdggnIo+e00js(X4iZD62JLTqY)f zRG*t^Tr34rj>{>Rg!))Qi`GR%0=QIq#0_5{n7UUSH(C&JE!H7mA@D4;5xWW=y&k=n9qT&$)%x~g&5Bc$L4 zd7fUhqAM(tm5E~gPuidh!f@4jKeLLhp{hH91BGC=e3Ss zc$VKX_3QgFQ!4XqD3J=(U_i?PzADry zN~+{EgBbO3Q~rBFk^i0urq|P@q;gGS8|-rc|fcD`IukHnHSG|$I&rzff-t7<^~agnBs}{Pkbx3>&2&Qq1IRS z#9Amla-bP_D7^%f8-=uxy2yc{+5~9qrzd@IbdIR(w1@D2bIb!(^KwNQWT6di$*uI&KD8FF5J#2!*Q5CK5vT2kUvlUi32YRk?S zCpgIT1sH#tw+hLcEZpR|X*1#WYI8i@P@fPR->Ip_n3)SQN6736GBc%%O_jbe41H+i@pN}qhS-2fPGUfNwVw>vAPa-5Y{-j1?g~>V%@C zmpFn#*A4lJTW2tlE}p_hAnliofU9QfNMxjMo;Dk;gOe*MObVg$ix)Dt$cq<=i+x?F z7nKN;Zre#GlrNAtiK>e~VBbBH{kl^AlCdw^RYUZop?i|nF#?Bp{7fnwi0V5#uLams zAs;DEQhhKV^A`hA?ONifI{)WN&SFpCggAOXg%QzdP7H8B->t_LS zIY=BC(=u_p(%X?Sic|2lrnaTzQQn+rt*)NDJ{B9*xCy;r2%E;X9j}{oVhjGjcc5Oc ztKagob_y~pMm$;cTxvE`Eeom?j78p;9gJsY(uRC$Qc&-vxl3yIGZo~V+JD)ipL{U> zJ`93d45?S~jYgx7^|hRZFHvyuz!9bovIVw%oK5e1Zm2#&y~LQVUM5_c4~y{9Ev za_pBURl0hYMX6w7RNASaM|A4NmAN7ARa3n_TYKUYDCM+{b5mNjeKTHZ1by7sxi1D# zfCXJd6TD?Q7$VMBqt$*gmLea5NUK07bBxg0d})`eN=zssAB09y9yfiWeAQLDq(ZGj z+M&^K!LBBTEfR%7A7OaOsZCH&SK6~wmWgqze!Gu+BQInPD!$Kx3nc-jT;J#3Z7{oCBZ}5=sq& zV}9!f>@GlZ>K;@|$z4wUnO~Ud+^4*C;}n@%b&y=)42V=--FShjb*ou#rp+(Tk)TxJ z2C2@@-vX7jq*y%hjtLGDVYij!r>6#K>?%tRK2zz5VxlPs7VjE{XN60T@aDI7isI~JmV zZH;n(viZ|VntEo6hr(5=bAF5N2z}M%xyqzUvJfuTme>RN*Vxjaxa*cf9;Dw5k{Lx67&=}~eC@Rs0LF&XxwEjfo-J zO^sNF@p4rBYe4+O31h-qW<;vWtA;IEj99zq<(5}$1|%{ZpofOg>nGfA-`rY-Dkq>i%9P32E#DS2`JSayXfGHZ?o32T(HY z@%+TPAaP6*>nY`|T|7z&h($*H)B~3c`mP8R%t(s9EN&r`Uw@B+=3)J?-jFp18ug~U zc2aND+tSYdfiUbKVHQ?i6{kDe2)$DSSIU4~HzE@$f}t~va~EsXa&H%$^*fI*OZHc%pxTHB{;2raU4ivpEPD<`0k5! zCRU5cF-@f#L6R9n2d^78&@l3l$o7Hig#`_+5*~%%I_ZkG(`uzoHFntNOBRYONP@8m zU?Cn1q&H_GO?{PXgW(aXgHP?&LGy4T?$!S)#}Vhlwn}UI zbbbmB7k%s@x}Qqiw=e!*|AP{Lfp1LI|27*(heyq{{Q7jeB7dsqp<;JSMHP_u9`FRC*Ei*jvAH3YyjAY9cNw zr=U_xln=F9#du8OmCC8iaGP8oVz=Nbc8LFqcq9}q_JbbWZB#+4*{p(Ar&9&ZJ+o<9 zB>+^;lHq{=l4@^iKF8$>e=Zh-6?N}hH9y$&u>NNJIm4jPZ6JD1qtULC$h(5^x{4XQ zN%F_81RPxtQq~QTN1)rKkuZ+-Bd;ssTNxY$U5XsyyYyZkT>tv=b0zY{uK&4GH5>FG zr7!>Ux$?K{#m|*$EXl+J@n4@S-#)b)^vj;e+_0N{F-31RWfFBxnlmw1Cx`OC^qhHk zg!wXSHW~#pDJ^&4LQo?$hjd78#1b@LqYQ~yjCLbE-*E5`bjO!=B|DzXV}dWfnAb4# z(2wG{pX(2Dix!n=MXx9E@0T65vMR{MEVEE510S)sZsipCk>|-PAq<8e3Go;X^Z+YZ zr_|uI>a-8qos+}P!Ew7K?t0k2PWoO6ppQHnGp}CF6$d&)^bwB`iJ0+4d5_Q7574mS&IdHz=k2B zL*luz(#fPJ-ox-32@>*kk=>0dgAlvtgr<8rtek?j(HQT-q4D=^rk8j8?_ATE&vdJi z{OvOoj-i)b9_baP$P@(FAAFPxzgcuWy(p5-8!Vor4??%hn0ugZ@uuU8->Q=v3uQ3a zq{doFW4To{g14_BgXaYMLUbClU*XgcH{KJm>+!9`7doj*>Y0!H%NgPYgt9*76H=*U zFaAh;c8UKKO?Ddp%eXIMyEE^IYo)J<;lGE$1hVrp6Mv8QFHImOJ6`#)f0nsH;uj{+ zjs`Hi_OPGfr9Sq&v@mEltDxCDu7c+A3FqhLaf;>p{5v^8${RMDk*&1&&T!J+%L;}v zJ`$VnE8;~XKTTIzqT~c!y4ab8n8KY1fp37ymyrEK* zcGYUZ#9y88Ha}J-$E!lq{PJ@JIt-5!!M=b%rwr!BCD zU>|H#PZ!4-4}&F6I#qCT!vCvxCgX}rX)Xa5kRh<^6Tf!wwR3w5LPP`NQ)HMgEZlg0 zaKl9o9zh}P8_!GI(9S}S;~CWr7J2{+JuLK?4n5#_FwG>OjQJ0$RQUF(DG#E4 z=<#`kXJS__a8i`l!%!@k^jA2`rvw6*U+KY=O0=VlL`x~ei`>`RIVK+>JQ#|(YBd_c zZ6zb>4wJv>2XvDil7|LJfFh$~(8B}&jD9S3=;c$HyQq@#6$t$ycF~(r!0t-!{R@R8 z-`)t`7Cb+~e!jJciPw-&8yf&zhhS{qIL@LP!bAC>fvN=y++i@Hz#pvzF#ss zZ`t0=cX*nc4eX;E)w9+lEHaC9Rt6_G_+Ga9} ztB#24>i0yC-n#?4>f+-5Z^d5hGai3aq7KX;VC9yRc$DAY12qEh?OVDf8FMD2jV$tB znMYJN8?(x4V*zp^{U||WYEAR&mqXmoE%sQrzg2@M_2Y~E+0DZjy+FDgYQY{thcHBL z!4@Vrs#bQp>1w3lM59P%=FLTR%!D+A2Y%A`TX>5z;;R>ll*<`%@FA%!iBrtnLDiup z^K|hR9q*l=zT~=p2X7}Gw3786qJIaOl`b-#3HHXCaJn>lxkJld3?@@R`dWkC>~ z!jw7qxLHui$7lg1m<%Yl4jgWin?f-ZoaAO^XFIyxyH%x zyCXCr;axYPaDYxZ`*&T=|BnFD&gE6IdYIc>4CX!|C|l$l3bq!|(jlL>XAEh#4qC2R zjd_?smd6=jnbl%8^M<*}YRq>{(iFq>82tPA8IJWeH>SgCTbM4;&jtbrM)P=cYyHHQ;)zpI>_x(|BEOTPNQ(WdfmoifcKf`bvL9hwOZYIAaDvgS*`9&*n1(Z?lkP3 z^>Z)4-r`$F53SU-2s^oG5%w|&8;%FFhPH=@F9e@yWnAx$p$GOy-ve6IQ6=0Yk|98B)I3@sS2NpS+pgSuDezb=C4a&A4=)iT@tnZaHv$Pkzqs^X9^ zhXbD@+)MZK;n1EV;=eM&Vy4VgNk~z%Va61d;-KRU4r(1_fKbj3e3-{U*cFkyTFT|v zMuMWwBzYac0a9H*GQ{cvp$ZO{C|TUVC-i3oT_K{=cjR7Ski{Whwn#vtPs$CRvO8}Q z!wKYWIOe3+vAKcgR$)Obvn0T@cxL0Ni80N|bLDZ(z+Cb`Ekq-^{6GKs&wtk5zpuT1 z4SxRlbTm564zWnts-~PV6pH!7MYIV*L{T_SzLi&`Q0QYOS55wAiAn;P8#!em_6OY=cB@<>4En7TKjq>-Jt z@`?Ll(!{NMdd@oy@D4KMJ9p1{jv842pF1L!Ox~;2PM*tLRr+_XNj$NWn484>T)p0o zxjM0PHGQt$Y{Oi={qUTpXqw#m@u2|64|7R0J7on^_A#vznUSN0*maS=8VLW(uOIEq z=o=-;ld`j>811Lk?uBV~+R2P_c6Ij)D$^U!h)y!+#dr!yOZ z@x&F7PH%Cf``wgX9V^TxKQ&0I69}I68pyx`m~Qi5nG&1Wy@)ir^HNFRr$&e zm~0U%UU544-K(5Ou$cf)d(2Fr(UzFmS~_WAk-Jf|(K^Vu939VZU0b82thnaP%%-yZ zL2%A9&E!L6Tu7UtInIV3KfWtJ8&({YO;-BkPKTx1utJ`jq-kknM{#)4Y$eW!`A*Mr zVti)mml^aa`}G(5t38!x^(l*=%P0E(Q|fimZGc16RqkD~cKmhJL!pltq8tkdIi3du z^GbIq*oTC&Jx~Lp=HAMy$`nX){=o?;NCwxa>mf#w?}T>&>pCc8k|wgAda?8`aWk6; z2{NMp6$%;dV+S$>-th0;l?ix`2Lvg)a4-nb0A@?$6^Qj6^=;l|z7+ZWj(n#=ogo^* z-%vheUkU?;n1{X%A@B-)|{PQH4`Do&+X7T@jM<^6t;6YK}4Z=55 zR|n`evoo54QgQ-^ieich!mgbsxoVd@>A(Niz|DgUF#aVFZaey zEkDtdlxwzKdt1Nu7F%^2T5q$echc)JAg*1NlRCKFYc=mFJ9VMv9Y|NSO5@~;OE1xB zn#^yBzJJNJ+uDw`EB^hmqZW<~c!&{sT;zq^)SB9TTi0>de|S-0u+klx$mI* zAC&l$zR_qj8ixl5;@?K2k@@$qanw2bOY`8UbJ%Pi9CTWLX|y^A2d%$=#)`%`@h74T zhJR@+{jKca9?6g5>~0tPLBx#Xt!AsP*otw83hqJQCk$$ii;V`EQ-4Mi)r+vl-ugLR zOU7v6FmwxabmCtn!-bwc#DR{V(fe;<`pL?`M@;CKP7a#~`dSEs;m0^+ zPmxgCA`%^H1v8`+R;QrZIc_x%4m*d3?Y6RqEz3T+{?n;%R(v3qtneW%S_q_b2tS->X|TgbM(2W^h}LA1GL^}_oR0*s}%O6 zJeWNf&fd!2DVn;{koGA-?GwV<%L3achqjkk71F!;WPhR0J^+n+r^^4O(Mb8%?1}3D z2>%56D*y}o66pp&Y@>hqA7C09^aG-}t^8w;|Fiem&zXZt{W%J32}XJOc(@GOav2h5hTqQ zs~xu$d=HU?-ZVfy1H=cR$UQAMdx&B%4f-J&i7OdAzB!bgxwItAKCm!bUAHj%5yNcd z4*WoHr!-0D6fiW|;pdUU^oJ$=lf&uTOTNbj)PL0yIu_m*qg#w#*k_B;w-KXnT3da* zIE-#jhtZQ#wLTEcE-!X!G5E8=;0ag>ARDeot>}=*Uks#Mzx|j- z*O(uR8uNVC&LJCGOtW50lP@U0aeQEL&Ev&28QtTLBm@XLL%Ai9o=_;dRe%`2}*YFl(a&j(MvpR~VPcx6>a7FuPR ze^k)wJeFI4^ufWcbfxUv-o}hh0UQy30HEh3*XrA(g^F|K&yTE{sd|SNTCEmZ0Z750 zC?pnQtrTLVv{7Kucc0&`>$Vr_l(x77bZdmBm1r=C>OIZCbAsCx=}GugnmQ zYw?Q3E9&+Z_%}VMTT?Z8BGxb&3Z`nQoeuQw1g@&8_a4p<3f$l8zCJogl+g}Z->7(8 z);G$cqHOaP70F8_YMdot(aDKLMHUq$q}*my6k{S`OPRwtNmp0_bHJb&=j?s<_kf{2 z+6SqA%-anC#}@BM+>boUDyyIKu;HFFD5>TS`{aKJqR&%NXI2L^MQCk5RS}S-ds@qqQ^)O5_S+! zlVzcmwSMOM{A@ARj`0bzR92G0K36TP)zUX|^|G=Rv&@=VM&9~--K?oOw2LcFOs}6c zt%6o&g%nhhm|jO~S|zPaEiC|yJLysxlPhYKm1rm_(lD{E)}+#fB~?Gtvydj+ED%|^ z^!2fiwMXB4buo&IR+LTIgz_KegkCDykS_#MUJ7JlF_6-7Ad43SDINHni-BI_;MdT* zyz`x6{GF@mF|DdcdC8ZO${rJHdz4o9C`&A#s`8Po^I=xKNR|3Xzn>}hF{R)~y5vWu z=trjPN4oGwx+q7cEJvpNN4ic&x&(+(1SDH+xu6ipw6CT=bC~{2`m1J9kaSs)bYT#) zG)U2e6%`06DiNYnSp2Lr=bk6kH}p%YT*$#sKlZMFq5p~{P^xIiPOkn#Y5fm%W8d*2 z%225PuBpFkK5-G9TKYr$Os@ZNaCp>dWa@t$95gz%{>MZ7a8@s@<1wLjXJG}88S8hH zRqrU)Z_ib^W7fJINSTuia$o!WWf+Z{#{LNp+}ZJN~mkYgnu^EY}zo z>kEsuh5FEx(?b=j3X2tm$w-TogvBbtDGI^~O1?_qMFZ?7$EeCty=kOsEmHxOC;-dV ze-o8|rK-PT#or{g-|Q8*o1g~Zc(99wmhHYPiph4nFaPto!ewgL38GkoCT8&K4fb7f zL&Y5dZp7u!mFkGN=;h}Mc0KgD^6k^#x?SF8jMlV0QSTm^GDoFau>y;tu>oa4ch0I< zQ0%%`-&LmVD%EwBYPu50^y>Uq@GHX}{xe;#yHL4RrrIhhMmbHbmGDooN~=JjRjSS^ zQ)U&bvWgX11!}CMi+L4RT7RDuxlf;^rpGvz>WRv}t3}?alcyt{ z646sQ(&nNgZLWBv%~?nK!=fWycv73Ip4H~e)B0*l04(n|ILp|WEqa{;y>1=wvjEis zRDFN*s;=hY3ZUMBg?bk1&8F*$p}`1^NO;$Mk49%>h&|ZzP*=DebF`rG0ioSsct^1V zJ^eyIMC!o|GLSArePeVbLA!OFOl)ILJh5#~Fu}yOZRf<87!yp42`08}o!GW*o-gmc z-@ERQu70ZfPp|4)wQBEY*M4l42PJDpeESqSw|lvA0%OlQ@H^F#0;vMS)RMHYjg^P3 zk?p0C0^(y~0k|RKaANKzxnJ9L3NgtcaUYooP2k_}g>&cufO5sI4`lJhfx8#(1cJPe z$I0Ig)PJsqMZSLdh**x5FYI1jo@l(fpVc}yFi6KwC()77@C`qB?ya#SCb+WcwM(hYRaj4cLr1?IBVboSRMN}OF=ntO7bHsXlJOvf}HtR_~i)wDS(v|1lDSnO7`+!C& z&mwJk*VD>l?j=Rl$4(iv;5hg=^d67>+1JGp(?bjzo2T>5}`KVO-)qz~AF-R!xK7#N093F$8tluf{+LMYE0a zNk>CmlZh

>bY!ZHjYLL^3wIGGaU3ls@m6Pb6)(>6@%-VJo>&f=K1-igLS&a7^R zN7k_`m0mOLcYKt(a1&gLRT_goR#}u;xd*Ts^i;6xW^`Td%*|ZtshlIpbL=SK2WHvB zd0w34lF7;Tbc5pnXOA@|ZWO0%5?bpK^D&z15g%>C%*9^S0rq=KCsO71rpoKzksXnf zMozYVfYfhQYsN?K$k2pQ6!qt8P?ly^*2*W&OLs>}=52+n4)fYG<(AVyTN=q3E@JxS zJLQ?P&dv&;;q(#xY?*T&M!ds4`OH#*%xg9FzLP!V%zuLLEKdhJx(~m4zRu- zxL-DH6s$(mAMy1ue1vww_&p;@^GZKHqLH)_FSXO#Z!r89mJNMns{v z46*;_4?|$1@fOq4Ce3&1(guui@9Qq)eWR>rOw6-TWn~Nd2@}s5In&IK4Y__RI<#yj zE5`mvFYdfM+TK|JFh(Kd;E}Jz@_fzsGAULul=#Jgs^v1fmUP6tMF)ke`X40{F@s|S zxN+n(U;t%O&};qSZ<)`G(x7>foW~MW`XPcTgMN+IK7;_(Th)s39}NO@8EnEfa=CTm*cB9eUia*(xe7Y{W2L4+3z>6vfyY`7=1ubx ziqsve$o@KGHB;U}L-w zGWMnj*>cN`tf?c(GWyQFtXOPOxyyPye-a{R1Ws79*upjb-i~VsLdh_-`^%A|(NT@^ z1kx@pcRMLH;m?qsI8gQ>_nO;#Pkgg-F;nm%6jr(Ix)%&t@@Kx$Lz_stn|Mmk1Xp;? z@r)k;ZMgWPxxlO9ZwRPW%@KS6+_tp%HseOBF|*l>5fZwLeg-PyS@=i!u(%+U@<0A! ztQHk;8@oDtLE00;dqzdqw1V%!UkD-m*x|@xtS>6Bz!S~T#iTFlGkN*q!c*{I1@la9 zD>DdMB?yT4NcjnWrTEbh%;&zd=H+tklW(cJz0l;tW+v%uzgrnO^Fs3{JQaS}D1FUD zk}A??Q&cpjIT_9R{sgnJUa%Svg4{yBoWdj0^``r*^EPxS`y2dmBYcvN(H3t#E{PpJ ztk*t0Gk;Abd9O)#eF-sQ#v#c1D=k?x1^QhvegL^VyYb_MHj7jFEHwgh20qoJ z^H#z#thX8|xVvvy4{Ks01i+JI;7QIwgn{?CC|&*d@0mud0^*tJ0p5mup~dz+a_R)m zxY6AGjl@{JH*R*Sd0@M&cY6qyr`?TZ7vs?r6V*Y7*xu|ymy1v9KXI#-xzp1d zM;*`F&ei?Y$@Pw7nDlq=$-m?4wRxF3Vy_bV%#QrS{G@wHQ;ON0=?C^J{bqI3HsGMN zx8)?UGtsXCIIT06=1}_OL2qy7hm%+QmTFOPOgD7{!@jqweVx0KOng{EEmo6-#A>Nj zmKa@cG!rbF{{p}bh-_sBp(^v-4QW4sv21a%<%;3YweATKX}_etYhFqV*;b?d3^U*) z2%_YaNgPdj4EMAbc6kOi(_Rg)e>x^GDaL>MAj=9?ulz8mbFq9ECf%_J0uP;fHdfF? z+`}xZD`lL?=#73Ae;iG((v{a3aA6jfGgF9E9DbW%+J{ZN`ZF(H72Z`Y9PxbHjr`Eg zQ6{H!25oeUejZt>nfi6@5}p08yQaR%_1o|xHS%G>Mv_V!lW4mE`GXwz2IOI`aXWUH zBezGy+zn4f1DN(D)tk;M;rudaIzY8FKa1T{N^ zDqk_wtqSeoUR3s#dp>R>C5zn{N3#KoPeAzfA^O3ffdy9%f@S$*^`br_%k~X;T4HqG zg?6oC3FdS$)2yrFe6`1soo18uG;@Pba}&o#Z*^sYShpWNwP|_4V?zLkJWum}Ro`qR zjjanaytQCT(V?=D7bX=bT`I)6&_+68C_Wc*->mWeU^F9ayA)XUD;9~l4e`ifwK5Rw_{q7#+PJ;O?2Q!JA&Ht$>`f+LGlBjp2poy*|)&bAwD{tNvYu1IsGF%r5 zMZYqb9d#=~uEQ#NC@*$?W>tVf#LUIcfeyGt4&uc%DQA{4w{c*7%_sDcx&` z@ZM0-VW4`e_}iBoy{=-S*H95cCm+t1lO_uS_54eBHiWvQv=5YKyyFAR=c@;pt54Ie z|J*-2a8~C0keb1|vC(KI@irt1d}zdiL$?9ZifI{vMU(+rwfC@$4x@()q-~3RX6!^8 z)OE`>W_46cXDuXnsZ?t&D6M)p?>q?JoD!gW9U$ovr9PxX-Im{qkiEy%xl(2$8uj@W zjXQi3|5Gpq=weIJ9KH#101N6@v8VN_?Vg+|g+Vvc#|Ra*qI6-AaWjrrw!zVZtp7n2 zm^6eE@cPdz#1awC7`FSYn_$YwW8-W8oC>LIt$dL`P- zbPK_loq9-F!q$He(Q=+-x0~)Xs7YK>?AKn}1$LlgQMe3o=EY;(GIw15qxZ zf%BI3v}{B)8ma`jhnB=nmK~pP(5DCcnSY29;dgPM*abY^aMTmPrTpEZh)rseh9$ac zFk3T?Lo6#S5~O1g7xYoBmJYSc%@w3uD(JhL369c0*A>Xo zxL1i`=As~t3W~M~l2`mT=eh7?uu~WBNs4G=wC9o!!hjW^`e<@mvNqoVuEKMDJR14` z+6C?|!{IqM=J;V>;g+;y``kIy!n@6r&V{1(nap(nwmA`x-^mJx0yoGCA6v*Je_}kn zazQV&bt!f}oD(&74GEb|d3=I97qKq1>5_0Uk(VuxWw`*qD|*RE5vy@8^9%xOW7~E1 z$Ws)6cpGdc4te+!>VpzqDBz1#Dh6DK7Vnvg#hsZ0+TBHWrFkYDLInR9>)0NrBajT; z!#=xBDQhTRj^ipU4x)W$NCN%qM+WIdom1zUySlBy-Geu^Q{}AJH7jf06$3nhy6=K~ z^ZZdQ=XAsi@+a#S@#YaiKIf*~9YznEB*%_56btU-hu6OaX}PrFpOz;jL7K@_#DAaY5JRz#ellU}uAEDPIF02-*gr&YJS`n?OWilD z0$zPSM%R!%zqKL1P{KG65Gp&4Y>^0$$uzE==LvnmI0BrZi9~#m07m26;6eN`78F~J zSr%y2|CF*t3ylDi*|yxDSAc~g!=ydRq{7^cKyXC6akk*H{pSTSnXQ?;1f+Gl!lHn-++q| z&RUD}^ojYbntQ6%MoRxuIe&e{bJI_M9e|0v@SQ*@$Ir6W8eDl2PP@~Z43N8}u5FEQ z+wszm!_pHUvH{-RWsPV0ov**!tt8c$Z_ZbK;>GEmt={;Y?(k;RyIWzd$Tzqojnka| zxOz3cnLUuPNSa}i;>w70XYI8kcRO(n_^e8+ZmDW(r;Jp=P`^T#sAc&{RE4U?1v$o{ zX=@!0d}-;VAyTs?yr-HenRKY~cK$Xut6=g?QCc#qIGOvs^iBRAOq_@@1Du;nuO4Iu zT_Eu-gKtY_1&;P^x|-t3!H42+SB!a3emN`{R>_$?Cf7rc7&Ta`@lgb4Un(!f<9>D2 zv0Y*KoDyy`XaSX-7*Y?`FQ6$kdG(-4stM@r3ES zmzumlnhj+ydt%E3L%asZB{uw#T$~*f;(R4!LF7QdheTel#g0g7@|=|%E2yqPe+m8; zLv>8ox&LGe<6s~TH~HmZz?oBBHZX|ef(4Cm-+%o zJys!AqP?$bq}7_kZiQP#?I~6p_7klu33rRsLl^o{t^%6Ic9O zTU#5kJi^;o1|5&&yo5lW3q6iO)R$ABkcOC;W{&?mL0Dzf@x8;_Zpnv7pv-@kj^-u&qgSDQlL#_q>Q2l zxdI0R#KpuVu#IXP2TXP1MtC+f69$f}w322i<}Z>7(o04FE`i~(uxPjfnpf4bSB}-h zo|alyxkPnaJwf4A-F=_QbUjGCX7p%FJa1B`Tua?87}oj6p9HMup*|$1H#aT=U-Cay zsCy=Z;IBqP(PnMeK`fD4-G-2TxX^HUioVnlfYSmtGGm^a08#G7FWoE{boB?}h>S=$ zq@;A^Gl8aAA53vDrj{01WvSIHeZ9!lR4jcAnT-+V`9^_Z_O_=1VQy?*#3Cm73cV7T zNF`Kzr#bX$xvE(0c|+lZDR+gjy?NVxGa7fjJp);^GX_R8$$2T7;{ti?%!@C?Xoy@a9Z)wO zQdf@#ZWr{v7D(~0 z3bM_29qFM%gzQ@H6|4Jp?)QW5W)6Y0!2oxcf+YIQVr1NkjpKas3^^SfmGb0~q*B94 z%7b5)Uy87>{<9;a6z5vLOH9s+@_u|2@qqGuRTt|gqaOL2{KWYb=o+n`GX z`nMD3ZSOw%ql+Uq7Ey0gz@C1l?fu5uykW%JTc?!FFxzV5(o8EMis9yf$(hvD7*1&< zM)2_UAJ;A1jh*PDBS}z$seuc=FPb`k#;}Z?uEtQz`0Rj>OFKMXL_5DH|8YdJoD|^K zd|@iQaJag2Pai448^pI!cp@a_qzexQ8|Ty*$RB2-g;lP@7Iu9d;}tJ$Z}sA6XunL$ zn?F!v5>lCv?!osOOE#R)ITJgk8%NBV$pt zenQ-i7}?x<|1dS@+dze2(2}F&+x4}R+qczuc)=JP>iz7v1Vlm_y^lQ1S31LBH@qB5?>MjAX8zlZ$Dp5agOJ%?r1~ z&TI1fvJ=`u5R=srAm<}}q<7^&@{?Zls zlxQPJ{EtH5>X`2&@6UBMkAVDZuDDde@X(n(Frr=gM|Au}9eUn7M*l}=tJ2a#whYb( zNAOar$)O;77#Z*{$m*?+L@BC*j&#lxMZj6QM-gx)P6l;^0Tii;f`Gwh6?wU4FaMwwsrEjD^K8T_IW$Ez#)FYWUYxA0A6T zh(th%d6jwW-vtLD@qQtIj9rS)N5`3B7tU0C0i^Osg{v}4#64v->+Gt(yOTY?`s8!} zhAQU_+HSC!vj%f%iN}vd>BSn-K+nG&FlEB;4|W!8iB%T*oGR&?dt_rGY&TebV^sbn z+i8K+nL%V)&Ln&OzLHTRG6^4Wyx^&?l)Z+Hc~W9DdVwF-5r!NzR-eY!UWXK-X`KPC z$1>nVJfSget8Q&oc5hE_r1IHdZU@u~4;NBo-7r<2pujYU34uXV6(<6F3C zi1iOI8)A0TAzZ)BgPhX)&8oQFO`8Z_`w1GR{5m{fjyAZ%&(qa2z7)K{Ct$t$a*1~B zBHZxdqfhuiepb`uFU3=crqQ5{EN#Q7qWft=^2CS{m|nI7@aLQP`+v$9j>lgTJ6deS zR|Z?-;(&qyOu6knwm>!Xf7}`!uZ=wU6WZrOCTa37KESH^sLjqucJ0@rz2Wn{o}~vJ z7QC7G??xLH2cax{`IleP1C<6t-&L*vXMS%Xo6RDs^M z&Sa?7qV(CPrLf-6Zd40D0(3DE`DYr&`LB$4>+B|2)cbQc)gPGB)f^z)L6FWhQ8>sC zEfiM0IfUwY8m#Anw4$@Iz?FrN3~r=?d7?6|Q#RCTZ}xBH0m9A%R}W_26ZrTeSk`=@ zdIX6{CEHf-jWPN?X_loKaAWh;tC%4RfZKoLcP_gNTnu zg8aVts>7J;gCUr%b1eOln`B@L`0SL&oSBY}iG>VR#5RMlkQDCTC+K~g{g0!DpP}gw zD>8F_fav|Cc%+O{+3?x+=(41I;iFV1iP47-gY@4$VnM_Ew^7IfwBI-UDE)2Yt;Dj$=mGFR>X!FZ|SsCb}y60*-YzeOh!`Vu8IR2h&s8DSubYZ&>0Lxu;t>&xmD1>Cc9wOV1T3tv*Y%c;3Es5g=1+eD zn-{wn?t;m7h6ce^>>maa(2V$ChrXA46ru%(xbsziTLT+S@F>)C6OS{{t>#6?|Ddtw znTtnwb)*=0RcViWYO6Nh5#!gSB(v_lpk!rz_L?|-!PWkLJnBcZ(P=1L;R)<-MJ8&n zr@EVs&^B<%e32!eZ4$)xo9H<#L*m|aHj9p7=H|CGacL~b!7ULq&Nc5FWK{6sFxo9c z!rt^XefnqI!%h0uk?(jJ9&xF!Ceehl-N@o(vy~besg-6qor}`lXn9qW+p`Z{or==3 zaf5%VCaLDo;yf0nyvlO&Z9^G%M3d4aIF~VbnT__AHaIuOY3?=!8bEZ=?)=@fFSCi@ zZi&T$=->^zH*>5n!zFM6z7h0BF6=S@G$5P(BQ6DboiKSc{(GyqAIKexnvwo=8Qj-c z0siD6I>AZ=vP3Eo`2*6R1yzgVS8*6uY}bQ06k>4WM(3X!EpQ`ednI znjGT(ksd8S>o|UkfMuerDK|68N{M0QC0{LQ{{dZgx%%&btCqjhHgBR)EQ1JBAIvX? zaned_*!QYwkn7UT({BN|7QkMRnhJ-;_ODch#Pj=Hus$8w_v}M_Q_2Fr{Q+#)UCQ2nt19lFNr+h5U1 zIm}aZB~O6Z&lZXBxm<}cE3y~ok0wn|Z_&3W#4#bCE0=CH1mC#-t$V?J?24nVg<49y zak9L9S`uWwear?k`N9#LNN*6r0Zad6yF3Xy7*-vA=kIJu5zGN@cf9oq^n12Zy~Jv; zVg!;7fF1M4YBU>$Oi)B>j(5gS)laVxjOjo7)M|mR+#WWA!xCY~_pu4VihSvQzH&zk~_lqW_VM4%!t#M$t z3N+LQ#B0;%vLGg5#jQ;5x*zkEGWOG^UV7=`BGg^7gdjKST~wi|Iz(NRqZzlL2XnVQ zFs;Hhvug})(U?lpJahZs5CMo4#ESxj+PlH@kcBS#3m3lu(P6JmRax z3B64XPI($R9kOo+%(IrisefG$^(2A=m2zO7knivKX;Rz;4fEb?yvDtDL-8EjqgfL) zzZdEMkx1|Gx1ctqYu?n*vRQav5AE*&%`0Jec|VHXTs_k&z)KhPXrJ5~cygo7y(xym zZ)b9KzPNkpn8uwnD$)0cD|Ow_ZeejEn!~f#_mo$e z5BN}|o{Wn|vvBVOO>zWmM3MO_#L!+p-;-Nm53djSr@F95;g1s1vKLLo^BV0Y6G3|3 z@wx*ajr;4YmnRV6td=IBTc_W>k00HkeY}hxU9l8(+_Nf3c5N&pb)4)-D!1y)aE(3cQKBC$YKUWLNpk+*m8Ai`Mye3a9XdGmVAji8%cc zJlDgi()`MT0hvyhP!xIgocS#lSH@`_?Y&mAc8>+WD*cmo50UVkw7m8EhvogNb!-}g zSBYtsLIQ)Ay=fNeF9t7;J>_gzt%tiI6M>7GJ;=Bh3HX;lfQMhc$-DArTOc%jO{wTQ z?wn|n!cIKqTa*nBD4=|SK>#nf%0vqS4|d5^K)|ymG=Bc)TQVR1`*-HYo6zrAN2j1bhQZD?9YhQ2e zr;jf-cdtBB$iNEsFg*HL#AC-2(i)UIY{hUjNC8U?eLzly_ZmlQ5+cUzIf$!BsBTni z@<2!_Q;CA>=PFZI*HA&SM_=a+qM?um@UkW~I-1 zz1?cimA*}+hHZ`*-EKF z!tQwp1p%NZgvQdP4onZp0z##f<1Kk}K>s4gj?x%l1BaJJxzJJU#Qf%X0w*&3A&MxK zFO{;^VC~M27^c6;V2x+#@d?;5?9Vzv!-1mjh+kl(qD1}m+ABhI`WvR60>!*4T=s1^ zP`Js!6M2ZI-e7Hs`NRDN@)L?Wid?9r)2n3{J(ce1=7V2o!&I23c?VrU@`qT1E*P4iZ>GOw^HamnbUk_kku($m;U98cj_3ZBEN%L34Red7*A8XV8fi|E>^HUqb zOCHb}PXIGY!)tP_K3`Rr)?e}*Cr95zY&a~1i)JlIlx#YRwJxoM-1@aG(tME#sGFiP z`#ZthBfr<2nYIJ*c>H$~DY}paQzk8KlAmfce(GM!S)rFj6DS?FpaZDBZ%s({&hBvF1NAAbWpMJ4nl7MUUPog)4$g51QOFu>+Fj54%5`| zu!#_VRNV%v@4>%|=ml|YiWvW3ZG3Lel{%O({yDWCniRwo4|aLz&$(e|1X~n$hjtT?{zR;KSSpQJ^~ z79qoXgkkC6XCcxFLaYj2YHxvWTzJCYL)X>FJar!B|3uv&)>>3PKiBdZnfo=DhpAZe z*zl;)w3lDcy=;|EG>m#aUt>=jx*+Lc&=gIZ``948fC!R9Ky%j|t zy&s-vn=4&%BK4jOxhCE^#PT%!0{E4bry?OAK~r_=_%*7RR_hbv<-vhkh$6RkBKnX9-D-lA7e%?8yxfg~kG^Nvg&--!v z=cP#o3me%q{|Pic13s^PXz~>PP#U+`QFzk7UV5yj0f5u>Uhk%>(%m6G9J8!nwF;72 z#rfC{0;+Vf*VPQr`-m!v3pt!m@)KZ!M1(tsE%e*VB0297!*0OooNF%DEzq4Wt7F}r z34Hhgy3o%8-wFwNB!k(bsDJ(92&OK~B#7|M+ZQn=nF$Nx2W;|g5_z$D(yj2O5%BjF z80yX~c@3-g7VGNIFwm#S8mv5~1-}E)iL(ZHegXeNK50qJ%*@CawVeI!^pF%X_0zG5 zpPoz0a^@1ufLKii*VcuAZ_Gfz^hgCAt&8b!IUz!?U?X4k*4EMUS$Ge-q9RealqA}Z zLSGy$IVLJ&(yF5}-j4kASV5>_%fG}q&5cSzN!gN}fmDPz=D)UAqpUnzT4p!QM-Z5F z2zI*fWnd0Qa53$L$=IbPNeKHf?w|W2csLvdaw5u0=}PJ3q{x9wi@EolnvyZ7KY}#p zTB>u-Up~ z|9rrTl>_6U^=gZWhR$zkyZz2Oi^;Gw6tHafNnr;Nljg$R-C!St3EIoAtv!?04*>P6 z)|ubvPVBOmeQje$TfU7~POG2QH6|Fpar{Df0E5b8q_7v5KT`eZH4+dbM{VBMXjNU> z49C}Qn{r9X<|2b)b`Y&;GozT*;%dG=QJp|M)1I8n+Sf3^Xv=4r3kC56m7XYq0E86* zm1v`*-Qg&corNO_0VcKEaOSf5w-VzdX5owr!~o*Xf7e3=F1oI$xZMP8=Y$q>JiA|Z zqqVlh)Aix6SopbU%XX|FpevRv!eJYBq}5K`9u4(yz!#zwPh>een_856t+$Q@MK%M> zWb?l^J>4}>!)yrke%UbUVh;9EaomWXA-4}jE4!9IicD=tLBj?nhQ%A_Ku^^xYkj5arY;ED61xj+Kk(6sPtXn=&P({B zK}DAPYgt2<|0#o8c+Vf3i=3K6EWGrL%ZAS!oZrQ{s`u=ra^&UtT=p%z#*T{*=n?Na zQ&Hx=OM9$y;ri3C&VfXjx8&l6iXV;Q#m*OGOqkI7@w%q$jYY3JArs&h^abcQGWF$> ziEnpZ?<+%97;1^E6&dka`LWbnPfwPe=A7dWXsf@6oCww}7hEuebU)oQ-v>K%h~)O$ zGw(BVfs7xU*Tre9X*cd)>^!#B6a2t`FVwr+ha4WYM|3o&{2E8fF{cnlpd=1}4On7v ztY)XYXo>fgqrH%0M*pLIFcQ=SkMTq7iGI)f&42!p^$yuDB%3wX;iGpX;r6Ube`~?o zyOl{`t*uU^jhEwQd&9pXJ_p45(I*MI_Ba_AX(;-@;P}+gyAGozG_)tps~`-d62!*us0f5r-|%!BKSk%Ukz-E7-w3ADa=A%!@{iDKN@bL-WM0nHnb@J z*cNlO*-X4!ayFd~aQrmczXEF%EZ3Z78zhkyfT$uT%`oO9GXDOhV9VDi+=C^|o8a;e zGkH zuTNYwd|eW1%l*;jNJ%`L44*F|YM%Nl6pfmbL3UQo?aiTV0kccpiplg{aW@=y!HJ@i zytSXf>8tjpoV!p+=cdk)SQ?*q<-%h_XI1S!(A5mM*V^#UKxy6sby0NQgU=%+!e2&h zGrmfDoFEZH0e0&m?}?W*>f7JW)*FMBGw%& znlPhCzRi(csUo)NUVXebFaptB?HT3ALvU+Br9Zy0G#7~GFTb11;J#5`2}rN4E8 zGPi)87xn|g-UX_hn(q;@A_TLi<`zIoqIzRGU_IMqXtv9jl}n$O)h^nSZ*m}V#?!rh zTpG$CH4Gk@4nT(~%a#k`sH_|IW5LHEmr(NUmF%RNVWiuS;eD-W(SKU%yAQ(t@rsoA zOT^OwHpa~O^)K0qnsP#xp>y*Fe*kV7s3^u~u$0CR!UQPhmIdqfO=u4fTVVILtXE)F z-K;%W+Tzj?oIAU`y1Mxs_W;swoBcqW>kR(@OZOTRzf1cZH{N9VP?B|uFcQOX3~HGQ zh@w7XPR4D*@;W63hN#y?eqe43p|V(W{eTpQ@(Pwz@IsTDK$y&f#E*lxgA;6D-jL$Y z0&%S@Z*EGFV0BwfdnLQWztiyT90^gNs(c#^6Z}QdBpc!6`ln!l5QY-&X%GeJ8Y!GL zhr=5R1@NOTb>}ioLUS;|4V91C1NFg!&!_FD%!|IGRY^ZOe7|CX>~E5+2dT`N030u$ zv9^X9pQn?lb>^MQ-#Tjq(GHrU{T^{vQ;%?3vU)Q^kz{r4P#&1L!TwMFI{&<_1~*FC z=2zv?PQrOr-uJQP{1U#ugpZ+7#Yy@-%`sYcTxUf~1UYs5`vkB$;N#unDDnAZFS{)w}&H=2$)~6@v=&<+CsV&K?;13NlP#4TQ*+Pcp#T9>7@i>QN z>M%dlqQeS%E-5nx2o%{d7btrD;{lfOs z?gRZDUBr2Nck`gfpDW*5X6PTSTKli1mY6Y9oB(`AKS@7yJ&Q-{Xmc$;OV{Lw=4g0S zK0zKalt)^9%$%=XUXJ{x<}-0sSmvsj=y>IL)UWJb^J&&wURm=oJ<$Kf`M-dhGlr?d z&=C|N;MM*UD{eQ7d_3U?QsM4Yg3x{YOru7q)j<>KXSK{X#P$jd6m63HW7Y2TFVa;u zA}Y{-$grnO#U^njIl?*fOED&FBL>Hwy`V5%w9tsU=Lp5e)|+KNq$7E20kE3eaMw*^mkoG3;$PG1=E`6(=y z2kxZM-MN|0t@>cKXcsII>5GQjF>m?9Gyuj6G2guPCI@~uc~E+i`I>E#Ny?!u`P#f5 za$Y0&qDM>Rkgr|(D4e4gY^Jn=L4>X+mu3LYr~pv-Sfpk!NE(jM zWvXZrR7DY25CL1=?>Bhzk)`6rY(eokA<38Yd}1>(1%;^raB;4~G<43FQ? ztb!o*!~rxR->rNVc>xz}=Y93#sO%PwmPZDc)TwZ+vCUbgyxT&>TGU# z4LMBiWwTtJXMYU%3hplbb}wU}&6@0lSi8GpuRWxxg@+$pWgCCv^5rqD2rQ(FG_sYd zkJg29@IQ~!GZQt%goA+B1;7Ikn9XJeZ1?{_YxIZ|PO*=bX%tIQiut@oWfP0nVB^bO3I(*ve;$4>&67n@+k50= zl#Nj%B`jMr(N2aR3$9IsR*Ykw9_cgV(wbl`lFn$|>v8#APRlz&AtzP)=Zqqbadr6D zOq9TeD>+(@u&LS?@e>$81EVetLMh|+3TCBCQ0@x4KC`%|837q7?U!K_wxG+5MS6#) z4dgAi?%?~HnHNo~7}$YeLZpRKK5K@jrJAjd;uB3e?xEfBfg1Un7*2R8x7rv&oOkq+ z5`{)(n$?mdySWjr6H}u6NHV%*DKq~Vvn1$MW-XdKNdgh0gIf0v_aJmoGD{2FLB zfQkX=n13Ms>q7Vkc8cjar;0kGLPo*Ag28-5gDVb1wu|O_glP$GKciQo+ZEZJ!^-j5 z3-Y~G^i}(^hQjO)YFNoI7v@^n)njc>)(AMOO9Ovoi4{n3*@W4uCySeK8BG3~wGe;> zDsCI^{Py0~_+4wl+1IwBPB!#}>%gsFO5s01Aa-F+P*r$6#JbkPKuP~NRYeULQQ4{g z4#}1}=1Y}$JGAY}v^uJ)KAGTQCPd{{C3a@74K$??D#I_EZGY3W&)}PoJFhRz;wk|> z{Cu@tn6hA>ty!G^<4tdQ<0Mi5|FF2WJ5^abF~r&^_hh{QdACs9c&f6>xrKvf-K|qx zUkZ7t0@*&cU+;xD1(UeHb`QvUzBv$pZOq0LSZ`@z;;B-YPzi1)gW8)f^*Wgoc>Mhb z{(~nDLGG=Yjy}4zjAh<3ID&fHYY!#tx`;`A$^f>aA(Z$J%=N*e%agZ1NX_sc@Eng@ z(j)pksH`i#^<<$}bpVu*T4dJ6lC);){|xTb?&?^2v2IwPFqz;}AQlQinoM6*sh*fk zQ<3aepDZUZPZ6v1;A*4(dPu{4UC3MQj)nG7mE@>Kx&6|n3uSp5(kdVAF9gt_W?z^o zZMoR5llZ98`U&i@(Eg}e&+5?qZ2R8(_`r*wKmCgAhW|K{w1IlH>5P!{Gc3{zirYd8Pgc;c($ z)>*-~xzIXU%D1VW<#3uuqyO2m98r{6(yD`c^s){v8sG&{jB*IgDho;2f!F zoX9 znzQu|4v;D&m}*`wmf7lrW3R0Vj6SK3>SIIvel&X>0mDk4C)!zWc-me{9pxndhXPYN zyRWFp{uWbN#JJZsuCYB}fX8uEdxI$jj#JGRhcg4!9MyfqhA>o|SQRsi0?qoEMu@^j zY*SqE{zc{HTLKt_F3oBqnYu@rzd2Z8^07|3o6u!Qq_PKmy)vYs0k3`nLG6#tp`q{Z zOLkxPw(wtF4f%Ou{d!Y7maJboNzx1>oJ?H2b#CEtCHf7k|Nzc`I@V`wS z{;m4CAlZpWxfxyTEPFGA8fjd0iP03bSGB)iuQnr@Z*L;REva2~gZJ0o@&?hBXoAJ4 zk0pPTkmv5B=*4Z`YKuj_tMNRxV==kvj6;_^9#cSImyP=i23Imnz8^N)7e1M#cxHym z&CVDMRNatiqNz`ZCD~HUOw-NXmjSr#5ANWBUwxBAiLNc$L~Dr0&PJC$sL?j!aq_Dh zHT=A+0(~!dLibcK-Fx?&%a>>S*9Jb1xr4yRn`IFP30p|azUUVYx^n2Ynm{2G!7za^ zmY1RFs>=-HHUKr#&H)tjRTz&@EJABnDA=D8Mof9NSwe*R&i*|cm!w^43eH%p?g2?2?L;b%a8F$8{1qX z;_&iSYm|1+v^(4ISZz2Z*CWeh7<~cz(%*|G&-IJsFl~a)o{o;yi@cuq|LSf}<`q_* zUhqxYR|lCNvx57Cyq2y0+S#>Va%_4$cnwJqOOvy8aS=l2JXABA*`6ga;ST_i+Gdn+ zBc1eSWxi6T?oE9o@MMS`T(h+Q+CS{W#67Uu;z2}ogz!uXZRIQ_^_f}9G;<2+_$tM4 zaGII-BlWH$pt*#bX?LnBZGe){D{xF3nQl2qnPm@h29ny{378S#beR7 z;cXj=EG;yig?kz_pv;gh5a%bzkx%cy6Jjmc+2>IwZpc9$m#hQFPEYO*6lhX=-;0 zOQqGA_T$~gr}d3U_NWbQSgoG_BJ63s@|&}5oo*(e3e26)QG$B>2gS2t zn$Bfq^Hi6Pt;&WsMlr;)xbvx-Km&ML^qze9_6aPasL&!I;QVbh<#kT!Ng?9m-PxH>TPXnr7azCdnn4=KbSmd2!1r z;*1wNZ@zaLJxL2_bMz4dDH7T(YJK(sdq70I(!f&ng{UL~WB7X|1ZM2D$U>9j(K3aH zxJL^2(_u{_^{QB&U#xF@5g_&|V)?Mz?uY9pa%6s+*e)P)j}kssrqc0&(Spc;$GDL^DYjAqcbK1ZG_j<5ao1WX?Wk61j-p0v@4T}N<`EAcGMP&Yp9eBY zQQOJ?^tW@9*CH1cZg?HeY~pr$=!hEdvhtur_anS=;Im9piNyrGt%nV!;qkzDi7-Cn zVunW3R zBLjW=_)Bt76M@#0I~bj#>5Hp>Fb>s%nz)0$Eksih=7YbT`*)R>H{5N*Zi7I%)+ZNQ zkH{&Yk~+mz>NvY7E*f`N0)lPGdsbjuBL{*iRn`qefA>KDb&fkCsxCNG!X5<7o=Lq&M%k7NraAZ#v5&p zmd3#Z59s$8ml@92xve z9T7dJ2{wYrMZ08K$H^FjpLbRbDHiN)tTF^=t;>g0pcBrK9g5%rQ|}e36{x&SP;r9| zCv5Nu`RPL;W-3-cf`|Q(oI#5NH)$rzA4H1gyI=BtNo`$j{Fs^|IC#?#U9*B2r=yJ4 zg6#-GYt~pHrw><;`+pA7X|6RZ2%1KGy@LZ=HOV1!aeE{LA9&nvWI`Ctst(&y@$v*%9x(&KiGPN& zz`+6oxp0dWujdj(i?Agkb;~!uK4D#YVHg;n%kMwGogW?TcqFPZ_7DiL2Tn9(85LtV zj#61j5_Sy)wVAPhiwbxi7QqF}*jHmlb3Pq*Np3Ju%T71VdA`~`@JU2ELik4|@+R}f zF!VR6NeF=EwOK)-JZ)1tGMXZ?MuiAkOX*=22MiA3ka7{PhMt#WRjc6N-8A6+KnvD! zG{P3j|5kVZatfMa>ooOs)0 zNLtHRE)qZ#V}|8G6sx2oiYTM-R^5=UK8%P=+-W?LS*%E@2YdNARaq3xgpqU{VytNC zuY^wNs&X;^O7FxhEL=E}4U1FQo+uNzbzExbiI+9){oak`+ zy2}Uz#;Ygqc4LU*B_^=g1;#zNi9~h8f=AyJG=wJzxSv{vK(t5qX?3R4e-9BIW=@B= z+&WB?By8`9*tqXC2q|`KPH%8Tal;~mYv5n9vJ4i_L^nXaOWf12=-)+I8Q(S;tJ%BI zBVbZzh2;g9(H-@(AVfQ>YkVV-^(o4V6ErJhisN|*U{=UlKZSGcWJ((Qo-_2FWF%HsaEpG-=UruKQ62eY7&b`>)YIF_<{y!NJlRT^qPoO;~Hmo8efZdrM zB^I1(|0Uj2Z_gP(28s=_m*EZLE{pyb|DhGh3oArt75DTtk&o6jV(I~Ww7X)2MDoNw=0ZSm%T+PmTN4)FDbi-%$(}m87=zchvu^awuGbW$er~CDH3&Hl! z?&kib$|wKpgXTcfaoggXcW+f3u(Td{`J%u(sVjb+3QunYC?v_+wRJUNf>Q6amsY0R zppN*Oe61;+if?Egq5WSwp=(eQel9r)@!zqh>slbvL-iw*Q(~+>r)$=bAC;yEY6ysF z(k&Qx2FN#<%13&h2yKW96pwTAWwDUj9MRkWqyU=7#-gz(mrR-XFpZKV{si^QTj^yR zv1~3w%=v5fP5BeQ3pnmGGzrYJN`rlBdy;pJd%g1JtV}yp2O%D<&&D*a+47YYCmc9m z#1nKkuiEI23yhImdG-OlL)QJ}-7hZtJsN+n96a1h#W)9sSWTlO9I<{Qr^KQmdgzbP zpR`GLz4Bk(S3uT_3xcjBj9iT`fMPCt-eZKcyjsXarhA8LEqBUpW$J;7FR zlhe)*S^#T<-Gl6w_R#4^+{|bZp`lIjWu;~KyKY(+2P=&-BTN$FBvRPWPaPc@v?OCj z5$1QM25G%cw5)4>)`XK(j(?vy(+H2WExO!QU?36by6|+pt}*|owzGGVh}PTCeqp;{ z2OU>b*-Y0(nJ3etP|DU+t8?&oOiK623t4J|(=@9XpMxP)(4sGn9hun{S{HaA^??ga zQnY}TR?*CmH=S?e+VqM>CJR*$)g*S%P8CD-m||?tP2OhDOXRvBbGAPBR=ysB?PRki zl)=zQQ$lYlZBl+$k&xaBZ%VG**R|9m9IeGy20{#~e_<2&h-7EKSPEoNV2ZC4cCX!k z+sN|#jZmaD1_6P5io&|Mo^s7mI^0X$)BhyhYhUwwD!nFh7&WbQs3Nuad-M$*V+H`0 zgN6ndpu=tTFw3+yMA!wigG)L#(lB4A8?Ck>mI!?rAM;mZx~5!#bWBVfe;Vw=ccmbj z^3>WYH}l{|cjSQ|Q6dy6$m@e4P$4H3-dUqo%p;e(+|9};-)yZmG$P~R@U1qSwnr&v z$8dM6vZkiCMl<}QTUX<_uPebQD{hYIUJj>&ha(4!TU7n#j9A3dT`#_wOp^#Lh_h9Z z1zAy%I6A2znm!1`hhIzhE36Ria*XZF933;2<<)lHmM0zkCzqDH*zslBhjn5SH>2wc z6gcWekH|ZS)dWI&0mC)-l;f2FzHR=_s>B84LH*JiPnw(Kcjo|+ zjgkgnv$7VCudl*}lMv-5S&W)jOC~FRM&0D_=)lLOyZ^57%mYmIeAWG%(a;dF7owO~ z8^t9~F_||uvSu^H<*kO-#>=z7A2>?0lj}7ms#BR4&Lp|ZW=~b8nrlOV3>MMSz?*L; zRn<#Vvu+NKQJeddm?KCd`snc1X96*_szO}rvQkt<*?LkNb!5czD!^@rLh7r`t3w{N z+ptZ43M5;BD3$E6nxeXvXw>_bc#<`$URX_L%-&fweKnZVNBP$~u?r);C^T1estS{K zOHv;t#K2~L6Dj|$8Y7oYPK(5wOis;>%q&VaH|Z;oN^gYJetYN6Vjar3U&aJ=CdIFi z<1mUAE|mC6kI}#YA2ro0tL(cFC!0hc{4qt97(S*pycbQF!~^jB$L@(d4%3CKN_(yx zKP~Fs`}OGHttI#`~~Xtm~5>o zWZ+?VfE#XE#`hBmzpWqA&n1MKOJ>7{ZySQLLTaM>_)sdZxereI&4&5^UDv_8hJpx< zluo0>_*X)||7M&+mte6US_$X6W({9ujfZmBRk-;}x62Ti2g9bM%<#48z4Ozd$3Za1 z$$d3OlCpW%X!Rs6ljML_dR_>*%z2MxA=$LyNY9A#fBd$J0FA;!TdAlZo!?$_{5!;e z4V9*@veU{O4*xQ#Sy3$`7pXy&lamsarHS)!U3sm68c%>T?nel^$47@b69U{wdw8Pz(X;9IFhksh2A<`l&&R7nF5T0^t?c`7D!8~c3keWT zMaI99=)C+}!}y?^J0Gl&p*!+lxjgd;c>3$dDe2A@EF~4RIdcC5HZC}2Zc6zC$K@qM zw9vu?xR4qdZ^9?i4bib6?DL`xNhCB0&a8?OfiB9rL`OJ#_WCXj{r%lLLKSl%aoWnR zfWfPu14Q{M@RHhM_?PR?O6g1888iFQjU z9sLspD@|6o2=l~Bu6=_SWZg*xQTNYT)72hH_~T0!g>rIxfiQ_9>o}fLH|dJ`3Nj#4 znKO;eWCW|M39C1)psz8PpXSqWZ1kUs-Al9U{@&f;;jQoPMET*3pAOh8akaGDGNQj{ z;6sww6b`Zp8K0GMZJ5%B;8QIp>ORlG5kf-BW8@97PP+4g^L}&o?Wqi`FRChpKtqddvm$;+?`xI%?hh#MUnXVvVJ^q z_Ve>0{%)@Px}D`AecJ8paWZPJMdi68QlE4G!H@5~g+0>r1DSY{LKCh&dS12Fo`NKj zIvu}lPB-Q_WV1J`B|`)IyMt;ReMQ_X2&3qo^2tE6>QJNcSQ;=0l3K1&`lF4% zL`<>s1yk+TJqtnDt(@3Fu=;XWEZMc+`RF9dqilVfde!Vx(sDdZ-8^FC&}U?k!oSfR z$w-?XgG$TZzGj}vvhb^ZV!h9%{u+uy8TU|yGja3Bkvzu1k_(!!DHRl9V;p>@1k&`w zEwco1ut6o5_;fOxe{&4O(6)e-oSjF2mS_XcSL|WGakE@U@-5Ss<*HYNvQ|dt`3m}Z zzns0DxL@z&`E$KI{IP8$#((jCe6P3vWNyqzNLM;gE>tLXdAYniI9WPq|9Z)fy#uML zc$RjB6k-ea0I~rIjYJuj;I9#@@^&1EaBj-1Lb>$C1oGE>Glx9fy_n7Q&JX1S#Z_%F zW5z?q%Wsk0{%vYd&$4(6vUWR6Y(IA?h^F{h$?Y?T4~4ey45Nlm+4o&zS+#)>`VN&w`pxri+#D@6X@@ z(SJVSGr4r{ub2PrZ1wi``n=hDyVq}j_QuZ{zxJ0Qqic^3WaYb@-wO=+BZRM&{~m*Z#0Yx*^{Q!dTu#LJ>01 zT^9CU;ArW<_K6_z7z01e`v4BOAH00iJD=^Vl{Y)@K7^2; zm4Ew1#CFcb9^pX`W#cw#fC|b*KC46l_hmiTW?I;+>N-9=)^J=^Jw5bCuPJvgP5f_^ zlmp9P9Z6141RoSq8Mtp6bR41c+@Mt%+Ry{W4)#^wvUH5T$8GjweJ(>fKbzVyIG z4W9Djc*gO5PEnUZwu1v@I9sF^tCC9|?rsqigUIDSzewC9>Sp9Za;N`ls>q#1M~p2O z`T>1;unI*$n)8uA8qU=IRw6V7InoMp6zNIM-4m2< zt&TS-F&N_H^Ic;%_3XmG<|U%2-MS&nyoS}{rc@fpn0GO->|u*Bz$5>o8(+0BBaJolH*3sp`4eOfEfhQ1adL@0{L0 zlo3#zwCeVsub|o;nsFE7ZPkkC@Mg!)yMOl6g)twgvILs>_q}ughp!A{GOBiR8w|ZE z&g%~>O`^2c*3jPp0L}p~ALu7b^Q%=<6ymc=4@-5DehsWU?V(V6HBjv3v@NbA<#mah zk%V-8ylI=r`l-3xoNQ80A@M3(>FAmcJ_T_f{oSX7<|6 zMyF+ue*#sOe&u`rqoeo~#iAi-=y|gzYC*W|U#ffZzdCXT-umbHz9TJxRi~UhGG>s_ zc3gu`Z&orP@)l?kK#COWdx2+4M}v+r+DF;qff*t}f>_=&U9&rkL-jt9vGif(1LeZRgwzVmN$cSk;d@F&Vg zatS)f5ZgEBWu6?WoR{SzvNJ);j4Amb%1>0f(h<>tD3M5RO zr!+gGYKKoIG^|A(Ebx5Gj%^!GYjQr_=O+akEu}9$VDYOclLY8NPyVd)T-7$94+P*R zgKXz@VPIa^#uGdMkf;0>R0*0vcM4IyRvqbVgjoN+7n9zXj?~cp32VsHEo9hMOAoL+ zdQ_ahp}xf)5-6U$&NNV{<4-U*Mnz6axdY6SpAgignJVv*LZEtc) zSS58r-EDHr8ku&E1Q8$~+n{-WFF!eCkh<&ucG<-%Hj~8Zp`mKZH`Pwu4=jgt1>)MZ zIIDb#AVVxNk*AmVziqp1-Qm^rDEL8-#q9K|I_hu0^jtDIf^9P)c*o8vG6h6;+Z)L5G}}e3ASN@QjxVMmN5;fMzyBXXzHya z+AxYn$6cA695c(M=1BT_S4rULAtO*Xc8tNj$ncjIrnkYng2(xviN4Nn~FPWGqQckvsn#4RnB`Zs;Wfa4cWZGg4kdQQk*wO2CmO{!K`>`|5O+BFC6Wq7os-c%1AP@$T34g86|4T?A7gc}+x<6cV>O zYe5gX*SAm#noI6M<#j4u(UOQVQkS-tH3}iz3mX2Kg*2JC8xs4^#qCRF6l7)dTIC>c z=I8R-&^E-+Cxk*CXQ#O?4Hz&wjc`Uf|Jh`g(JWG3`WNmyCb}kQ2)1%a zk8vo~l%Y!drjlJ2Vsn+kS@-C+u=t--Wu-3Sl55%(pEx;MQ0FtW?~pk&RQe z&ItRuFgyg5Ik2&uF4&_}u4~Mg)gTBJ+>GHcb=g0qbL3C2+d{VX7;k=7Y+fAZz@b(`nq?#MQHA1!{R1C}>zs+aRgKpZ};s zIz1|Wf#izDr8Porw>RG`c_qQo(J<|oX<3KOrH51>QeD;?WJF&s5=&dQ>?SO^$>-$S zPYzdv>uI4X2Oug)=%UkK!u1)4MV1nUw1<0RJl1fNiRyr3b_QotiwQ>rKOl7?TVelG z9%zb2=RiGMOSp%-6iez3q4_?@V@dJe~{8b@>wqnv(3vg$$#!_grnfH*>G7j5k(wOc=+ z?)9_?)N=kUMiY(na_Ku5G=U`tn8pUaEq4fnSUir0+O)8TvK(b)DCFnScPq+)#+=F_ zz|h8z%)Degmz#Hu+Zv&qPIK0{l$iK$^DOO01o%NZ1@Fty@|0#WZ2rYCio z1qrIj@LLxokr9n(qgo--b5Cnb*Roc(+-DE`k~3)8qO=Qiu>A=9+O`SHHp_t48nOSkyMVq|6q?Wm*r56PVLdFLE=RQ=k6E zIDkjK2oU})#8efF(a56CnOlv3tq;QJmE*w~>FP=F0_+}#KDz#eP64DAf|5%T(!KCt zVO6D!N6cE!P7<76Cl6Q@OB0M`E`thEdcpJs8{gmLk8%?7slV@fmzt>HYAY;t_pz`& zH*GM~P@Uz_y%>G>I^1MZaW2~ik}9h8Z-SGJRxTJ1jdaa&!gl^p?W3xXzr_J)vNOI5 z$w?2@NE)$pA9pgkVLg)9tN{*IBoiGXI$$jZ*&RpgRzb_DdMw@~f2hwaOlWaA<8P?> zO6oB70gfR{)-`1fLmo2i!30c0Cdl-NpWh_j$rJ|0fycAACTe3)LSRERBQ%X@hyg0{ zB(9q5^}<372{GJ^s=&^F?wIIzR?|_72oXea{IXBPgyCf5{W&&5VF`UHkIsayid_1K zcsdF}sv_qg0o&1$EE5Wa9vXKFFUAS%O12KBbXAlW4}sJ%-e#rx6Mes>`r8==wk1{b z1~5m$yR5LAX{aX2TPcN=?Vptb#k+M9n+RLz6sU!kqPJK%e_>Z$VVI zyGV%~hmLr$!jG2#va)pvTJFfzp!9XfteiDH8F|Nki3psQ5=0f>I{i*uBlF>?#)V8K zS^dSQZ*l8OLuM5epfybIqXyXJ?XYRI%D^_GHpW3%gFYG5zp3=P&9dBS-IoEZ7uTZS z@EF3hR16X_eZeq|t1gkhCuqEs^HGvNa0_OpOjW=*zW#VCfd%)Z5YKrOn)u=1DN55q z!oRHEHiPVSSh~iCndm?xv$ABDtq1ZK;BeLx2qFa`vu+kdGxi^4X!dYU--B=tgD*MW zl(HcjVZO^z8}PvCp*`=GG80HrF3)gZK{0~4#5n}Oh}dMy+o&UXOthN)PS1<+f@~8Q z6r7@Fu_QEZi{<0LBbHF6!oF(_TBX$;MvSf+S63ZqsqQYW1FCKpY zCFE3syBguCRm`6Y@L2?LDx=YDr3O)|T;t35CvD4!^>N;su8Bi0;;7PL`kSd|cj}4# zd9z>F|3FP^Bj!AKLpPWd@s$|EtUw-_S(Af+xUbZi9!-{ZN-<;vRfuuB^{ z0g*U4Go0wfvLYUpttuKFYE0q>m$U6C)wB~u(59tx!!gJ%lAAtR&W=YwsE0#;VKA4i z17|E@9HEop%Jl1=&zKyl5(c|XSiv%F0qagEZZVR50_3wc_pGrarU{QyIA$B`1;8EP z8c_=xiHsMoG{()DT1G4 z9q)l!xZhArkX_~$7C1}O9Ru9`tR`BQRZh%r^Kee1J0W`<%X88oDX__;0csykaG`!*%+^Xa{TEMaJz_UOS^x?;&sz(sZ%59J1K4Xhnmy(9mgcyzo{LJFLJ9BJP&1 zPO<|$@bFKvzYP$3viq=)ttZ`IzBRV+4=#?3=e>ETO=9A_!bqXwhq}BxXcP9Hu@pGoz$I0J@WZuuEb^eOE&&??q zmDq6`ccc(}e3N`+wdD*+T~X9}eN$nU#6=LQopULgH{a)y^%sYSeCgI4`w_wOXczwZ z+;gu!*#&!hj-z;#av0yq7veir2d_lKXi@Ts>WX*QN!>J<+E1mwET7n2?0^AEZ-5Lp zdTF(Fzw9mZF)QCIE9BH~42HKL*qqi5pN9TiUP8_>tt7$?V}QY;MHaM&U39K)1;mRk zV#-K+2;zAK{Cka<2 z5R}I=k9FI=Iqh(BT_&gJYW!XqdsVpewdh}<60?YM(4#gnF*LVF(ynNdlwoC&*Nja_ z&`T06XNfigYHJ8syI*aMjlJs<#m0_}L1+`l#**8h7QFiXLAAU!umxD&K*YjbaBwIk zKXJ+g3Atc{)_)akVhtWus-m!_#gtInAyNXNGzq-RZQE&or~ztb%J5>dmC?p~sc?nI z6SOsPWB(vhlL||0okcE*8~O#eEf)R(MU%$l#{F7T()1Jtnwgtony!oIKF)r``4Fd8 z(S#o!t{Ng-Q2Q!vI@6|Q9)|p)Z9#=XoWec4q!XUKO;Nq<&_&Ykil%#SuS3X@4AWva z$u4wr9in-FV`s_P;NJ8#n`{>0qh1C{n~0aUWtzzSY2j=dtOb!4nRZT-_cjg@ugb;H zF8e|w*L*Qv4OtGFmVf6=$mQFXw1m05!$|pzAGw1aShqMui%#p5*0=%5e959JWX^2E zR_)>)i~Z8ZA(B>}pYmsi1d&2YDNcu4fplgIUd zyDGk$&R%+)HyQ050qNcrq*S55zpe&pD{l0O$O93_|CV%U^2j_M-krVXAZQ@b(?}Hm zureAca2mv5xlD-aSX$;sf`i~(J~oRl0Jw@LRIW0P{6+vz^HM*e*1A*Ez{RD5jfah1 zZ_9XCB_zEaPX`tg-}rXWi`S4wMsSpk3EW$`#F7H%Ma~;$xdwK=pefgustS!hK2+K|aC*c|iJgAB>JfmStO{E*Tf z!d>r0{}1Q{IHM$mlOcTO z^A|%s#XzQC&Xud!SSCSbAjK5VFmU=VXMgbct)q5(vLVM`4ROoo+jkKq9TS$zv_*tj zf_cjvgAVvfG9>J!MlG%F8Uw^(CWzGn1DsRPF9iE=MfvY6HRb^SV&lXfW5VOh_ga>} zlu@tcEmx~pMla%mAtVEXg?V7MxoODMW^%SPK6(J z9co}7v6!K-749rVMNR9G=6KU*I<2F2`^fJ^3K;v@6Oex@%SZ8Aj`+I1;NtY=CV=cr z28{WrE=fw}evPu6MyGnOf0N*1!*}K$*p|2RJamkX?vUTP5a+_>^oGNG`feed1h&DN z8^)oP!o3H@OoU_S`Y4MWE^B(733xEiNL0|c5=&`J`JAMqn^o6_v*+ALZ{=5VPj_orG8eh!d#tiu>|M8ueuUBk&x|NR&dN<-iFfl1InwXSa z?i~9#gJECr#f1C{WX}{Mn|07D)xIu6u2P$8 zc#HO~#N#~v<~yJAOG5c5kwi%m>y8~kuY$bZFuc!i_1)nxt{*?WpKU7mIFUNaMRCD? zah|5?eM^;RQ|q#*G*s2xd>heO5y7nByFcEEMY?xqbU0%dwE(fxU&~#Hi@sO!$LN`w ztjgz>H4sgvqCytJ#x!m1ihGTeuijd)2B?+Y+)%fV5gfgk&L(t3*0+&Tk($TG0_hdX zCU&c)$g&;6dzk8Gm~iaQ15f&Cca8)T{PMXRf!ai;nPdTnK#urlAvj&5R0 z{Flag6Ehr>4t}~l1{o+x1V&aTPX7;!qm$J?+|jSPj-{WXaI8(Gf`_AF*TAuqEhJD$ zTl+P>V?Xevn=ds|O1H zKkSXCsJ6j_mCqZx^^cWu2>qPzR^MO`WO&{^tdFf2BPO?T5&w(cRMx%Q$b@^VdiY-_ zd+lhorZZ}zhwtG(ji_Tz!$RU=Vj`tvcX(M~+j#O``Z~~h6I{|w(`?RkGu1n-x~5gf zFHUhs6F0)`gGdvvgMBgEJvsK^hgk>}Ivreu4Q9o19N8;0{(r%n4LSRWm{|FpbKu>^ zuocp1Ztl5`%w*Px=4%?QbtBz*C7#$)KVH zXBCC;(ZgM*tRz(5uF_Xl$V0H0soFEF3OZ;i%d73GwOBOvK9QNV%n@MSk!xy#F01R7 z`rfFIFs#rJN|#x_l66MEx1uWRjSky9JJ)z25z=_)Y;9fkWB>k}A%gbpbLYvdL`b6c zYE~|e_-vm97vl423LhiK@ILgkDa~Sevniu zO)KB8A%M;*PB}B?BTP#e?u;zUT>cw>2FFOw>l{8&Z$k1SDJH#;DP0Lkn}+L$dTP~6 z*;D}?uiB?3Gvv{Je@`zZKXkKx;M`}5`x&tC9zw6Fd;G(Yfh?4r&L*4MQ_=ud>k;7V!ld0f|G3&QvEkF-Q$njNd6Swj4{DQq z{w@HTX7cFn%zIl=_DH)mX4bb6nyk&Q`Zpid)M441rHI??heRf-9Y?mT+)qL8zze8g znB3H5%JPyqT@7bj5k_2C%+=*hwr0pL6KI{1W_lKStui|s9TsOD@Ai_)W-SFNpRfd< zTNY`)M~t4gq(>EV%BXEra}Z!hJ8pwo8cx}i&Fy?=6O?`@ycVIvPz;x~Y3n^3nV1Z^ zJB4x%XwuA_8l>42`@f6PKt4IW?y`;aef4A;`pIh{_INV*g<|WwmXt?I7|e^?`leWf z^t&z$etcZ{K@)rH^-{^a9D&N)(NcMG(s+bQ z7v|HUNS3)1u3*~kr}~=zlG8W&sn4aW7U##EkHKvP+t00^Cq|co_+c?77g6m|6A_+J zBGS0!H}di6J{1Za8L~#9GA&<;&;uAWN$tPEt#k#IPi~ ziu6{9LLrgNn^^qwTc{(9purS0OC1xsnH!4nsr*%Ing&loXOy8(OJ_Kdb5+&C`JhW4 zd%OmXrSK>hwvfLFwkcvrW64>aYeQrqvt8h`g=5TGE>2@PGeS2fQ->qLpQzKki(xz! zK>pbQj*hLP+bie|i2l499X1@HZTh|v@XXZzpp&7EMll4Jojmdsid5!iK0@0W4sExH z_9O2c-e%#%9f3WaKw=r=sLQ|;egHbs5Sz1NSYZymaAU*HtPvYU#cK&&>?NG-nC!EU z?nfUzfpHs7thI(%)2JgG6LK@Ba`XD<7LtMBMvD82ty!LMfedTp+v*&LqJrQ@y?>6s zfNy_lWxr|L34R@bth`p5?9Lr5k)u<_Lq8<1%9j;R`q*p#KEpnq z%0buZ6llj4Yg}D3gs&;ily+Bf$UhI;R0n=nj68 zPHvlxGj^S+LoaKy$fH-NEIL$rdL2Cr*O1R^3)Hw|JPlL8Xx@KQP9Vba>(V>jl<%nZ zn4H>Qee;6lk1Y(RXQLbv=KJisSP-pqB?2|oYA1={F2?8_fFM(vAz_@6T)#GSM?4J; z>wwFs&B20>923%3591K-qKHM9hVeazj>EK6#tzdu4+B6KBsK&8D{qt;$@rF@J}%-E4fl&{u=q$x0-)b|v&8 z%NJO*@1)gI?s?`;xjEZW>#LnFs=LbSx)LVLM$A7x zXjv_b7Ig3PKdFE2SzJ!sBs-|ZPmouJw~v>>%ZI zr}C!1+uPgw`QJ|N@UYkClfZwdnk*hUQxXVkzs1U>SJ+Q_eB=DK@C<&+&ws)GvI;kX zYV#M9Klf)v4G=KuLHn&CNYXD>yf#s!2FTkF!e$_ zk{NI{84Rs_&&1zNO#%Z+&wb-|HHF>d6svcsBh&%pYQv5 zh~;chv$A+Wp4455_{*lU^RJmYV#~U&TbpV8A99Y4@@p6LJ7@ZiuzEM}&$kIp|7B-1 zI~6YH3`cQPe*^~jB^PPXQtOVhCaBx4yvF}5<~hwjs*Mp#U=&L9P~ zVD> z!bfmDsb)pU!Sgl?HNzUpujF6`R=WwMbrI8Ix8ItOPP*R9?I{2Djip04x=p>AxQip1 zLa1xF#(5SZw{M^Cv0tka&O9$QExQ1r#|aivVJY@?py zCa?^gwd@LKGCoiwg3q^Ya5h<&9?$xEj$KV1#GoY0$ojRBVzi=7bW(*x^apqm;xb6o zcAm84SUAOJQS${^r4H9mc=j{cjxev~DV-=i z!OM616<&$sKat&dT|dgdj&oAv>=6rPir-q!RJ}4o#T~&sDj9PxehPKIVFTu0%fOn(OxL5eC;DOixHOrQ zTq!3>9M?1iKUNtkXe8-Fa~}p)34h@~EkJby9ar&(iS@{yo3BaM)Z#YS%HgQ#vKF+o zT$FF`^9O>jFYVzYM4n$-xallMS?%Zhme0rB?1xdg!RME8E=0){CrxJ8hCWQ21K&Gx zx2Lg!%A4*Xl(Ch-`NZNlK`3e?x;Lz_ca-xN*GH2f!wA`27srSQ7%riJ*1m z6qPCx(*xkWhtunB(#k`4BqxW6d;{JAx{dm|c?F}b#BlLWvFqf8{domM>wt}6{D1ec z-e|||*LwZM>HlfIGvT-V6FjHwhS#wY=f8A_gaKcV1#)yF<1t)g884M0fwnTEaLiiOHLheN(cwUIVL;jLTt0w#qYe=xB;fh|5{LL+RSB_(ow@ zIjrxHmm^V&`|KbhVdeW8V&{N0^~CC=W-HWH*H@tjhE?!357&ITaM(yRvCuWQXkZUo zj=RdG>~L?%O~U{T9zh`^xw*a-XUM}Qyz5O_7o-bqyMIueV&YHKM;vv(xqBz4KY?uM z!43%}Iq?n@iQg^8CCPj3bIzuk9+1|)o*`4W3KnMQA0qyj-_QZ476~+Q1#tPdEBfXe zMAI%x4~)`ZaAchlu?d4y&uD%lGjcWK%(oQnNKFmyAxsPgOOIV5GtZO7ICEgKNM3{B z<)?HPHvZW=@9OQ7MJ-%GvA$yJ_5Fv)mJkx>Z*J;CT`41+QZW}dROtQ=qUxXGURI(;?NE~bu>z7sAz`e%(7@4EnPo=;JrK#jnbH$(3cXbC0AAVbg(PaDEOElNYfc{;(iHO&R(Pf z?9cwc8r>g1s~5PU9Wm~H8Yp@6#P;PV0jZOrbH@*$sFl~JSr}ZkrV0Z^hca5#dNgTE z>{NZNWQMI}vE}(izF{N+dO>v=pd51OuW;`AQsN+jWO}AzB))wi8n`GWL*e>o6rm#o zV{JPg7VpPXT6*$R5W1JRU2#aFl*jnj?wqSJgPkPug$)-DXE}s7l?=*L8xr)dXi=>f z?1?zPaJQ~|m!kS-tQ5r>K;kfpRMfu)VN6+s;d1QrRq(U@hVp)Y6b3i>DjFCoSm=&( zpuEjLb#tSoNxC={A?_B7GZdrzGdPplMV#q4%o6uPm4(G1;N@RbbMl221FBDu7>xMC zmh^ioo@dpor5yT9?^nx!ty=nwiWw#Zw@=TY16I>sPPqFMy^jdx{3;FEekwvB zVisf*MA5MZZPrT-hkWSLvo`w`Iwr0)w%g27&(OlR0Vjh-!7C)h>dDA||&WHG{6Z6`tccMq|^WX14P%_FxRO7rmBUtex_nX>e8 zRBEvz;GMMKh@YVTJG?&}!~(>8@e>NBw8&5ElV&WyOC2lK;EA+B1VN{YH67__Lsdy@ zDU(RN!{FWzO2s27ham|co^GlE@#Drpw!3Cv|sQyAg-fc z0UqQJ-JORjChK^iAc#P_$Y?t-ME{Q_}zcoj%siq0{=uzmKG~zG{ zM+C3{Z)MH}C-sc|&p#F{&lIVX_Avf@{XDx+^=URe>mn9W2GB_{4ZDQw=R?i(r8N9g z+L7f=AMm(Mn;X0MG%miK??va=|H51&t<6*Pt_2+L7d?dWz~R&FWBiYJPl~6mW!^8x zriqAO7p6I+zGN`ikfvHVhN=3ayMWY=u(N9?2~s(>rux$a-87Vu8czz&O!}zkn|jnhm6bU~hvWd(|+5 zG$L^ zDRhWh!rCYOiO5HDOx#0Nmb@$y^y$>lg1)OYtF{NbXWJ?_H-Xgk&yUtqgLqc28AnoPOcLBO zbK@2nS(tv_??-SM`jnQ`Eq>8uW~X11^QxTEc%^jc)kA{Vh*qRA-Ozt+<)>1%=bZbO zykhgnnZi^{tPsEbpM;temAX<+i8ro5B~4-c=S>GBu<=Ve3+eHpY-;S=m-$=B|`Z(A4Lt4pq;^(J+T%G(G-p{pvXlu z);Eyqg>|?SdFAndn2eZIN|aU^m+eJ@4521t-f#MS?=02nd*ql|t|G-u?k21>X%JAC zzvlN*=AcafPomahCv-w-FuXWbU4k>a9&_0Gh^?_`Hv$*p=Fok&?_cm7Jf_??H)6(S|NMtRsGH)1d&4dYZTIB22DvV^YctW{!Spww7SdS$fUTX$6CGn={i8Po{M z&%9D^)OgXPBtwP&1&Kg*zcwI4NMB0(Igh$0oG$VWEhoV#A$1X-wqjz4o8eEvmsRw;LpGxDMS*X|ZF2Ctl`DQ1Y=_xZ<01-x>aGtD=3pzYW;Dw3vu4Hdx zh83dXn9k1L>(kxH84~Am$~2d0=!E&1?jT7-pTZpn2V5XP)T@*$pc6eqYZ`(ayj|!b zlWm$;>T%FXLyvG&?~kR&%pr{7Xe_hmK@v-DhX?4bOnIlO1?#XkHo>c{9aJE=pjIfx z?Ys&UDy_US)SS&Rsy$1*s}jIuvzc<>t{nJYP{rLlAyOtAt)fvUHGM!5(`=)aoWyIv z%WMKoPHZlvrnp}t#0*+e2B|*Ru3U{Uvth;9=uDWUF6ny#$Lwk0GE@^0?25f|SOjWs zLuQH8K-)0emtX`hUnL0m%9gF!r5wv$cEFkS_aT7|7eg#5FUZV3fC@2Hb_Xy>EB#ZR zlal(JpG?YS)PuNvHGWcrfIO2TPJ5taHMqPX721#<>MGl22R-sqY@UipI29%9X}3)- zkv|`sIW8Z;5fO(OZqUV--*U_=-Wz6mkqq)db{l@E&4JhhFs2kQodhn4eDKnkj-wOm z<{bBe7XUg(IKTrTw#_ECss_B|+A{4+PewfgTc6;c8+Do}PISqbL-N`C7s2 zm91fhMOEagDPY>w-BH(zsqX0O#vmi1^xpYG3Tg{T&eVXqunj6yXmuZ+Gr^lE36}0k z!4s)>4y8XyZzUq%8+kJ9A_M7+DVB=AvZ~EoE~0k+x9l!|2uoX#n{oaLFJ+-VBjXBG z2jXUIrhx(aR4wM>$s{?-uNM(iBAh>l2Tks?nvq{w-)ZX4%j{OE_@UGso3VDWPeHHK zP$5uIYTaq!qkK{?+b`Q0QWhIaL16h%7L)Q+4P;u85V<**bFRi2hH}&Er!aCWWsbp9 znr|r9+YiJQ%gdYumnG>EvU@aVWW<50Ljx8^81`D=bQwdNLgmA#mh8Oz3Y#e-FK}6l zK_y$jY`F(J#v@;!5@$C0nMPtVNPDD=vh8fpi|-5U4_b*3CB2;_iuiOYMp!;)=t@*6 z#3o@7Eg*N1S-z(!Lg0$yl8p6s!U%_4&+%3vN>Eg{opGu|KVe2ex6E#f02$eNmjrs} zD+%2+PDX9B!<(>qptlS(^$|fN0A)$w6uz4PAmCN|I0T2U?3Tzy`2Vx_?%i$M+T!5+ zJNy(lQhHid^_gU?t}fX83fbWo$PfGAzJyn>FT*JCX&S%`Qa@vi#0xy{#NoHzB(+~~!bxjy}NSce}p5qkV zLx4psFG!|q$TRCYiS|S^f({h~H4m9B#1#IIFS&ogQVE>XZQ`vL`s1T76JsQJ?XJ{yy7w=AI9;dvl_U_SdkYx^;J#lu1iHLe zSp<4~%QGehPM5fHylEcCIAHK(*PC(!oy!ZQ9u*>=DpA~;PDu)8eQ#^i+iYo3d$780 zkj~vSkCiQor{FSVw^ev!`x7o$TsrQ|!WobcAEB>eY6qkvIgG7-$LZ=+UNA%z8Cg47 z4oH7PeFk-SaLAjjk_y!`2Ac{la0bx5bmDQlum_rCcxdyjLo7bW+)si zC%DKZv7lZ&X7qEFV-!m=ZyK*5Y?ssHY)dU{@P2hm1^lfKI02sOD>r1gy}is#~!3 zt*pW|7y(b2$ixP&`pZx-b>%C}NSsFzWJ|$)q&TAPQs0LWPcLa20Jl&mtV;^;qhnd! zc&lQwsyf(3P+hDcEPE{4q|Uh9I&dNw2itPRgz91 z2jHfx`r&cK(7K{e>ToVs)pE~NXVv2!!D*P@l0%Xm)l*J0&YX2@5Yptqqy>g9Ibk#q zYA~>jHA4?h5@B4JUrZGc<3@SxXNrllJpGg>L8GT%5-#)hQ%t$)sp7{ua9UU^{kp;L zb3kt7h`Ag)%10}Viz`=O6F=w#x&%Wv8cxa8ZRr%Cwn%#s7D*1Qe!c{m-3WqV{i$zst^|~8|?YIW*Dn}P}yvTm!(gY;K zZbttb?TZxF<1m-=w<^b;=vgA6QWr*q=@Ryf?o+vLncszSuXgt zBKu9{LTz@$=V>uv$^u<-E|bJa34>X#V>!T4K#3pmJkWbWQty;q!VTV(&7kpP9=Pwh z`=dNRlulSNYQsDOuWVJ~mP>Cq@2vjGA7CEz46L zIs@iNo_`pUNs=HdBut?gmONc5Uq*9H?$U5P$r`$oqbSU5jS6wKKcM)4Y&s6eZ%f2> z9a0I!_ga`Qlw8a*F%(;;5klGh+s$t`9YR(jUaD9F`9iNFdGh9Q5zGaJP}{<|A_cn! z$tsc}HBv17xrtz5Hl&Xajv&A1=bHB4A_(-^ghdHUg_kAEOuNqzJoAg36b zRt~23#L;h!hEbAi8U%&X6zrk$5CSI@CL={X(EJZnUHO8IM$FF!r1O@aDgfkbt(6#% z&f!0E8gXo%N zh0hOwr4u_WEYv*fCj5{+qy;(5bQ6z^*}9aU!b;&_axV>@n`PQ$lWfP8X~MHC?~mH? zW+M$?Jd=G-Svm$2;bj_T%m%z@Upa0p+99nCsvAjM){ z1tM+8=u{?at;huOUZ=D&u>e>I>WhXjAUv(8B}@m%Q(DnVikl4`2)$TSXXt|UZr0bE z4ue`nEMpP=PJ3BBe|qMvddhh9jqtt^o|yTAZ-P!^VyaS-(e+A^S}HIokTs2tlowNl z@}Qltb%og(%Bn}7;g7!&u0ee<(e(vZ1c0zo(wWceD=&Kls+=_#6wq&1gj47Hbd|#0 zMeXl>LLXVX_TpRN(j>V?p18JxC4%6Q`auKb7Vh2^CR=5Py$@`a)4{HStXgi~8{?{y zuCw|0XiUyoXA)f?7bS{(ALwhz=IKHT!zLUqNjQ2#T&@J1l^WC>9!OA6Uc(vEMkC?8 z@d}(g$^}|*ZJwuDwd)ocA~VbA+g`D19xhs_TXx!8X(($0->zG3ZE;~OYm|*7vkkk; z2(3pfq@$wrguxmEugt-7^n21J3g;(l7;TH7HT`@g=_v9Sk`?esr&tcf%Bkn@RGe{B zSC8(xuSZH^I|>wL44=L--fXbS0;XxZ4q(0C4h2hahjdh&+RjR~P_3u8_8I`zIXk04 zkTN08&N}NWHIo$yd^@D0zMqA`YCu?#bvvsgqmJ3-y~Js97)9R6wsWhF8l02vbTs8KaLdMnr916>$*nPBu@jIW6^=YS-w_ za$_<}7fEs}9y=Foej5lY0o6!HJ4fABqq@E2G^mUYJ^s6xf@bpH+~1#i4pvKnFaK1z z^y{qjS4gsZWBXslFGaWzeyQP5+Aor^FI2Er7cj1Der+LkM1XURmWw2x@PjO1rQg870gWiNjd# zZn7imL%^lMVv}bjrL9Y768@0#JP`x(;V~vpdi=5M?jhd?+SN5tIhvr*jHN>{0B70{ zk8{CFHNo_O_kFIR^{ny&G!B+%cy6FBfsw~N)?O|@oxGOg?C3bF9cyqOI%Ae~KDHST znj*PrJsQ%^)oU~nVb^tUu^hh-;~{+k11*32Xsqd+4!z^;R*riq< zHdtN_V5z${CJ3>jH&oG4GuSjupP8fzIJcUh>jHJ#DV2k2*$%61z7~u~SwRJYXuLd8 zHs)L#XA88PUj>bXR3e&?#l!ZlxiBhJoP`k^ z+wPZ^H>`2Bl<$s?56Xp(k6sop(mdW5KM9s9N5wp11M>JpzTJ=KkE>A-YTs{p-02Gq zmDyE&E~#9(lc7jm?PkJnt5v_!2Va5ny~=VVke53Gr6G+J|6$|hO)fIP&0~BEKDzo! zk4vUuJRqCqD$kUD8j!u+-R)h6kdeSM=8l?684cdW(fovSxXOYqU&Xv*3}@AtOCIA5 zr#qWZH5V><%v99q$NGg1Yb{^EO-~m5s9x!7)-;mn$$(3cc{eE+pEdmkJyt=$Suxebb@tVUrr4s!402 z*hIKtbDT#J%3#_ugH``J=VtlXw5`KhAzSdA(k0zhSjkk{{UJ%SP2_mj*#9+kJQPeti+hvJP;h zt~?T+*I7^9za506l%BY_<~wAXxox=g3mHz+tkcwB1*;9CwnLp%!OsPABmHsFSQqL+ z8t}QWe6x|;hD(=l#YLF7GbnRo@Rq4*?2bX@;!26)`3)jn+!(})JYk@T@YC}i%lw`Y z5%Bo9`mH~`1cIrS>s_$T4)(}`G1*X!YPwZ#d67pj zI*Hm1YMaFYk2Cfuv+)AznzuZbVtwf_HXlKkL;9PWn^ipBkxUQ60?GT0+$f?x^P29; zif$057Ko6Z3g@(8HQr#m75Ve6vi6})JFL0tJbDen#f=l70Zd`Vq zS}PJs1M=v%gTs^eXRnT4AD+G4e{=Y`UJx$*J^Jl$hyPeu4Cy7bxL<42DqVa#efQJJ z!Qt~qYojm@)Y1l+d&C7Vnh}8j*rZd3l3WoXsf;Vdh6$qqabwb7xB4;gUZlGVnEyesQ zH5ys0^1f?iJF3d_O2$Mv%+ZCYGBarz@n%@xihEdC;kulA$*QAp>cC(XG?_aR04c zVqa!MEZ>v=<2(O0+~WUQPOo=97hl}vwFdEBc)T{C;dkKh+5pvEdkZd^&fk;s z%C3|4%bWO48mB!an=JRCyc4&=LRD}lZiOw)-Nvo3#t{FD@Wyw<|1Hc2xQzdM`m~z= zZ*S*;|GS^(tMGr?{jcWHcXuTF4m|ai&}`D;sE1*m3`xJX*IawIci=SF-gwz(jyOwH zTnlWy&6eIfc7dy>kZ*P=YbbZ$?r*i+^ot<9hU?okJl7hz&_Al=&-Wr^3C zUFiBgYl(^Ubvuu>#x-?)|AK106}$_U2_JtbqHP#$o@%*Ht**pcR~Wjz-Nnv|wOR78 zQxX0eis|zrKLQVldQg3I{7{?p3+ zkL}Iv&CLh??>?T}-Tx?up{rj5sfDR)WCx42=lMKk96Wq`RXU>sO0Ga|mV{&c0%WHp zVq`1*DROs`-+o)(o)zS=fe-zK4}ql|6q!`)Z@+w9dF11!SWN zvITK>er-gnjvkpDrGw|B(;=03WpeQx3DODU8I3|K9;m|YI=OBQPj{$7y)>+JQ)Iz4 zh)O&qbH=RcxTgnjh1ulRZtdgwH;~+>o_9?3@LaOx9kQ13|3~n+eKhM9_Uy2IW-QNT z8znxX1tUv>-`ouw@w>RD)yhcd1iY*85i7XrN1@1ADO2PM{`o7j88!fQK0{|oSPBLII$g{7X$E=6Rq*RB`9CXgm5Rti)M`)1XlweV zLL{GUkVouOC@^HRYO@5<8vCay0z)f7Qk)hhUFN~bOH7dn$s6ObM~)#TI#Iii92ZGx z-|3{ZP-)0UJT>LYS=Kz5x}{_LN}=fC%`%o^W;>i&R4NSfFoLYC3H2}Nm4NN6EB;}zj+#tohSe0bjjao8uDC_qnC%m@sgDP!u-tf!hq2p z7Ek#vjyDrN4_NQ3Z|)VIYwrJ(!~K_U4!vn`ojw-t|E<0L_Eu&8_n+=Ood5UobZg1I z9cN$X1l8vo6`0Lmzy6?;A%9$hD++^NZyaWmdnNwY}}4<6Yk9~#8E#$llJ+)GSx zS0@2ouh>t>!3hZkp^@NP?^z@#HyTJgFo6sql?>kmM@g7bod{gk9q}mQmzW}*#gg2Lf#Z^Y@SMb~FhcGCw#sZHIE1X+o$M+4^F5U}-|z0F4yd zB`EAu!eJW~K=iksdYj&+*RMfwkq;T#qYW%ede!*HWGA`Mmv;q#!UB0zGCzX@mYnN5 zP3J41wmA_$j7=#jNJ&f}vyc+1y;Fz^S)zuRuoN^HDwFP%USLK`CP3SLu7gFNFBz`f zF+9(#3s-n^R*<#OW1dRuPR{GWU(VMZ=e@OXtQsm+%~?+a*608S%M||5@ML+Wo8JoMtr5^wMV0anaWS3MBDg^gKSebZM#B(W#7QQ|W zGZV&SL}wD6@{E-PAz}mvh|Lccjn{LkP>;x*=LUOoo~K5rZd#>UDh6=me1qVOwBcMz zk-DjKvc_gCmKc>COlh3cD4G*LVl$Q60XgrA0R;hbEN!%uXYM2uWH zSHzVMQU7<#zo5Xm;W*C86@g>hySZ!1LSU5K&}+~1e}~~!!mN~RxnL=T(E)2 zv!m~+C~>@!CS6iA#w#WSS($T{lKPp7CH7+Q%LAdmEh5g+!ZRFaF{OYY}z zu}DJPqeU&T7Efd$y!VDd{MHTb8oCPZKy6_^6OUi-l!vBy|r_jm+LZ<+W=ED3s6?ME7#S zQfBQ$oQotMMxmHgbt$e~li?`z^(yfsYlYnc(a)nSl!r;77E)Lga^J-~&@7JnZm9-< zVU`3W3VoPuRB2z4^Y6CzzN?Ny#X^*0F+Q;+9!Hlj%nl!!6T-Mz+EH1|IrjOpx z)Z!As*(xH1%k`Qv)G8>_E0mG}_vN`d7lGY8H@Xhlffk}lP?Ulc%2J0hvM{wEMv_+B zdJaXmXQ8BdI)s$aWETjf8U>v73;VBTDwlw~;=lvnD;$f$$V!-wMq1F6+is(_qm**8 zu7`o;QBlLFle5siSaG68%?cnzPOTnuf&04JA;hXJ@S?RoT4%%p-M!KS2~7#h8imE^ zx=y~>RnV7<`arC#j~H->1%#HFYZjo@>n6}OB$+)|@|4oEi94FNP{<`b;ejIJDf3b3 zHP=g6^LYAf2RUF$=aRLs^b3@X@+=1-AH|*9&ruT=fCE6WlIp-pFai2#N_cQr>lz5; z4T3ROY3fZ9|K7#I{Bx{|#stJ)4?VXYK{iiV0e&;sz!^gLQa-9^)I(Ids2YY&f>J`A zMRM8V!n(#vT$n0%C3r8`oaivPu$BuA`uuuFnjFlHMt$Aww>*_^d7;^wG7AOZVHBeB z?-BSh(&QY#S@=5beiwdivVj?k4_(`4>x;xe7yv;%8w&QXGB3?)Ws81OdEW z68AXTu@ zHW*D%)2|3(5pA>;lk|6i%W?os32Y^#Cg#^jsDPfdKx*!_YgHqxY}hd#IYqHFV52aW zJbO{NiK|H&4bWB&n?Tz(RFp`=Xj@|YEf68gw-ExlvsG_iBgQJp4kWUEo-i=DSY%0S zK~~lQw2-N$9Kb-W%&RIU74#K7nWD>N2!8u|{uBuTdZtL6(LkwYXmdx}wV}Aw)nsOe zbL3kzo}wq5_L2>a+NKi7U(U@nl8?}o;bn#^a{l3C1$p(nVYU{@0v~9%rH5J@qYc1X zQnGX&2P}<1OJggJj(Jc~+#@c4>73G-j*)EJU97iFcthGUqf5`HLk6Z#(9cSUESM)$ zY>?2bJn&J(1IDga(|VO#SYcDuv1${T&GzBw^kRBACfyiBBvjhF2&tfvfYeHjc>rcm zJVM*Y@K_*j(mcNPP->b;V>wj%Bx6b>Q8*2i+eL|`vzAyv{+gP$W4U1K@^3X<`4P4@ zf?HDlp_a1No)vKgngen(x5K~ye7Y5T^OS-dRrd=zm7wy+(x53jtL^z!)}0pmyppv_ z`wtc3*ov*Aoq}H$u0n7?>3wI2kJd?jyZP;A36z`7B?8zymGz)JZ4+5Ege143a;~(v z!c?w>qk@fGFOXy5lvg{-jQN2DxV`K}aM2Jhp=_f$WmCzh`aBj{O2O6wx61@s!y_MN zwZZc828=1wFb+gdKqKpbY-NL;(`jUGi3^Xs;wdf$C66>gwo!I^i6Z3v;Z$NwQ*S>{ zQ@Nb#9+r;1b*xZ;CrZkb~oa`061hKh|l2RSU2`dVQ zWe*!2WYpjT78}b~o&<%Tj5f1FO)wr9lYxOgWrW3WSTeYUALPIg$fhHjfWlV|3u6(k zZz%Y&zOlF^Nfe?2gr@;ZJ+(+2=gIpp8Nf!C44Y(Yk#5!**JqNoy1HQVD`baXAV2Jb z`x0Kkz6_(lr)dB)Nd1hZ;)zEb=ajL+b=UyHNHKL;3IfVE$qY)o# zud$H#CO}>Xqyxc38jFCt%}fXOc>Lly>3dsy-ln@ro{_${-IsqpZmi11fIwV5>#JIl z^0TCeV7DA-2c_n6R`?HW+gi^qMuTgvs8rN0#_ljqP_?CB4P8Q$r1-h%^*6muCn%h6 z24o+d^pGw}vMFUdONXx8op9N41CcIF4z38tP217AVDn3!2H+q+NvPwR;3KZpmE)Su zEQSE)Nl(*bXDSbC9e2+qY)ZpuAkVlgq{16BCjR{wmWhn|7Z_bz1v|k4^1Gu0poomk zSnP@kW0QH&)?~!en8pG7_1~vVaFI+{%H&8*V-w&6f7u#2DkNKd#5%H_9PlEPDH^t{ zSSx1DV&3t#yv;xJQFbl*Z!uuk>0>ed@9*|^tM|XQws-a(=>L5@UE?K)ldEY}0}b>% z2YrgwL6Jo&1Z^S`9RgFy!bQw&LXiBbUQSdf&H?Fk9LLJE{@HOdp74l|=ch@^XmC)- z5CihzqvK$pPfTU6B6S*y-J6DK%2O6JfZasR3CZXfDYx9;gd}+s>3rcwqqjUePMKhF z<~R`_zh*NQ4akn;DEfNBq-b{}C59P|2BbfA91PTi5tjCLcD5a-67lx4<0NUwA)0PP zg}{KV5)Iw1aVkPz+^X|1Sp%b&6kXinWucJ)C9uz@Ly#qL|E@bVM^n?Ar4hBxe-MU) zKsz=XvET7?k@4xi#VKH3p)N^zB#&J9t}|L!d%zG;OBy#3;jrC;8Z{mRPM7?YM$i`C z|FKTH;?Dsz1zu9%k*-2l1P&p28sVWk=vj2mQ*x=4NcYU|{9V`&Y_$2YCRgDAE zmw?D~z3*-H;ii(%A?>x2h2h~J^O&TZ+x}C+>5`O*r6TPRKdr!Y}k|Jt> z6`NTH)Q7;!0A8h#xwfbEG6;o77P!@CR9t{_J-WEK2@hOT-}u_YEA}aK8bU^cise}6 zHaHYMjVQ87noRgs;0L57LCTW>`Ou+J)cLp|xW5WL-58I$5{bGG0nU7%Pm|-6OJN+l z%H|0hkdr*l!YNaP%5l;m^>GQED(^Xnwg=XWlkfKrtlJUAZ31QYl_P?`6ex}&kxgxt zFIp^W3X$cM<36`FTlf503Z-mFJE9jFovJsN6z@GKdxT*-w3$e`oXM>QmS>6$g@T}o zAY*~$C^yQZ2)(1#zSMjmj;Cyjv8WVw{gYtCr-BX0{?Di6@NcK&X#Wk6BRs(>U<;&H z_I8s?+oPAlqf`1Tk73`j3(Yioq3aN8p#D-p^Z+L z8~|M-FW;UPS1-V=LFM1q4%&6{25NaHhy~v7FhmjPIZJ0MHg*@HaQ3||Z`<2}-vaU# zGwW+C`nL0^8*s_!drF04icRNU5DO3MdmtTA{e(fGd=eB7tIOt5Ye) zf>>-yf>{KlR>y`?7VCEXB!?Y3Rllcf3a5L2>)SUWyhBa%YUQ2cW(*@yIAgIoB8JTA zE+TX(8?ltdKC?vxnuNVeo?b+p2Cf+j+$@^)d);GbR`&wGB$Tq3ADD`lk7q}3!f2FD zezhV)0xZZRfwSoV-ANrxz8%u&C5!0%l=(ak#DHwegGkaSo@eIeZh5?tOorBg&F`-R zn#!O>v`SagL!y`lVC`ftUE<138#3`9Ig@kw__wn&xs1-vp8r?pKLww%Gh{H%VEaAl ze15Dd+9A))D=SR`dG^dZQ6*lyFeNmx(dCsw7Y_O2tvu&D)A<>Kt3W6UD4s{bEb{5v zvuEBZjN`?N-=015fU$W4O{e0;i_hz1I480Y9Ms>>U%Y_K(_MmjHPQG#3de6~qKyA3 za|`;3_CHDCBo!|EqC+zZ&l?KA8-#tzq~IMXD~+<;usD-2Is_e!B%|fP4=ntLnHCN3 z7n~S93Y+*C+GqG6)zOvtLS(1)Hp!Y*f)F{YYo?JiKgrK>LC5S&;3hiDct)e&Oo_id z?{qdAp#QKv{kU$mT>LGikQU%pgHAR8fR#a-@__y2c_$1a*4fZM_$3W9^HXGuM%M(l zv}W{YW8)f>z>|=b-xT!+@U>jrC<)u^=pv;O2@l$dsaVosG{V zZq6j-wU!a~b0&|iFdm-?mYogf853V<9bFGhSWgPVH*ES|t|kTE3IGw}uHN?kjD`^% zMr`%=|3y~fWf%wiazM7HMxGcn@Ufa3*xA{3w49_O{~aAeQ@;K+Fs!JPVYGSF1=%v* zfLytXLLcYDDD=s}K05Sa;FfS4GjzMzKR(iN!3nuwbGv35&ZL}i!Hn6tR9e{#d^lsO z{WVttIXNRNo`oro!NF%n(-5y8frEs)2(Y0}=P6AD;i+8t`o32-VN*f{OdKjo9Tv~j zLE1Gu@%uN&XD^RV)HLYa?Dq#i`V`^M>%5EH|;&bKydOYsQ z0_Gbo>z8nKji>W>=>Z6`rA~y10BW(U2c3N!3io{K4)ZvO7>upl;j1uWO~XYm>C&K8 z&UHOmN5SbIy9hEz%2?v#>fefTKHi7LMG&16Zk5*}FE zKR`&Kl58qnRYWeTQs~>9;?G(Gc8bl4Iwc%6N$7WQHwUApH!1R&?Al9+ShroqFVyF^nM=#6sm;t!e+AF8O)F z%7`h3y#YN)DU}YRc@blQsY}us*GJ8(FcsO)VK(^@8*;iuuY=Y?PSFt!n=N?Y#&8pZj@A*>YO!zFJk^As!*0KNnHYw3Dpo9Kp3ub}6Y!o-quYQ_wSg zHi1X1_CvO80VZmZN|d-J)(4dYOQ88OzQ~xZ<{}?5KZ_v1mg}bUlJsa23VpM)2w0AZ z!b@ot+v=j;%xi6;4wKQaEo@hpYH#_g#00oqYIzy0Cj7;u0 zzx~#G;^?>s#l(;zVZA0QdK{*FHc8l&q9>8-V z=MsbX&?JF@c+SrZTZi(Q>_ZMp9?w37SO|Pt3Zw$Vhr1Ja^8xItdfA48tf}ioFW;WZ zvT*q!Vg9=39F0nTNkyY?WRPBtimhYrWNn6R&S})O*mlcoUw*6H#wPClV+x71%RPBT{?1dhR#-v|VGowr|?owsWdn%v0WRu1e9`m|k_c{Jq}R zd~D-D9~G63I_i&FV&ld}T6wM?SVpW&S$f5;ioPTmdc|#;66z$yT{#I@XPvB7_*1-j z)L+b@s{gNB7*fR8ZoGc(wiN~Oees$4wS~L|&kf~&pQRa6?#7J&4v%YY-Prw=<|NCZ`$m2YNG=KO6yAmOgw1)#m5VQV1ToI50QFcB{F? zS2@4RiN+fbZQEiAr79OOWu?Zm+^$$jv`t7!!x@>p{|bgBklVS=n^K)i&^zX%Z?(Impa(IX2a3v!6#YkXz!v`_kHwE0dHyg`Ms(CkBKKIlUT=TTVw6D#If0gH^@}Ig; zmhyfA{iKNHxUJv{Y60F|;Tn zygrbMtnlCCq9~n9fAn+VI^@{qM!}?A*&dGEI*QEGBvkjdTXF0VQbVzUN8N?4bRcp_ zd;`%^ux|NQ?X`x}r1bO(oZOIGXwY*1&h=$~w&yPCzjn@YQ$_P+ zL;RO}c^>Hhm#6=geQ}qR|CW$E91;KO&t1^}HoC8$W&Yn!pY|*Ce{*+t`+@%7$MePL z{{!J~Ap95&!!=}I>0=hddlLcGc`sK=za)`T@s47JS*YF zd(w=RKg^sy%%we-f5SDHw?x!e z#7l|R+%bCo^UKf!wf*l0WeX40TVUmjoqX-ssXEct_?Cjhsr$Q^N|d>)MHgO{-03X> zh|gync zM`W6d3dup1^#Wj% zDQfLn{K@)*>%v`GuqFwG#2zV+qOawXXnBG zcQ4QHits*A+y+9|<}}d8He2adK6IWy?j-aKP(RD0=!W~>$_BsO0bud|?{D?Ds`>wR zw;t^O_w$rBDv2oeWf45LX3&bqdyUt&ilf-bel11gl1<#- zyzbdkPPJ$kttVl;i_UE?0+T;B2@pWm-m7=cmir*)3tIF(`KJe6P&D-JJehnekGQhSo~pnbi1;h_Wu! zyk^?-=`U)twN`T{4JqwQr)1@8D~DRoXI>O--O&x;H?-3TcMajCj8!emp%kip*^02E z#y2kryTr2fP-(niVNWqLT~X3-zUoUET`7&nY?WrpZ2dw4S$m=KwhfwLx3jqsyOjkb zvVl78m8zEdcbmzc$*fy-VlL}B%GY16ZWO1LtkizuS==tF2`QYez zb4SmXykGyCuAXhk_(MB;{_cEwuJhCm^2M&4H+q)&e{NOn|J&P}JAd^2Z#QGC?q*Ea zc$WD8wD1KF{*_>gas8;pz1jMUv`FIZt}r(wK5qhVA?_x78zDQ~6fdO(esY;1yl`YEOr7)tM!L($Mg z8IF5#JN;vtKyFI^#Y&O?@L59tx3~BDHTu8v5dZmJ9*h68m2Bml>~glNDsuS_nZ)(g zb2fr%uXkM7=+ObQ1KK**bxQ|KxmIy*l2w(h9O=fmS%FnFnJ-Y$31?J(Y?t^7S8{;O#Q zW$zESum2>^4f#J4+4?U0KX&$ZYW^Qjw;uSv`*<4tKlJQAxPLsje>}K<+{*pq!Ruo{ z9-KXXzt3Hr|4nWqniE*2|KG0pe{5~`E>8W{(CRa*P#Cog#Yt$w~q$@4wC~c<^i+*>WPeIH_QNb z)BOOcKvutB%lqAu`|tl)#s7D=8vpC*ga5}pJyreB?ehOUaCfCmZf%Tq-`5-RdL<^0 zeacuYXG!Z}Sqh!AI17EK{*dwY;c0qqNdM8yxuOfmGWx%{S>gY-cD5hlf8Wp3uJ&uq z`-q_e{$=U=9s3(*^F$p8Y0NSP=V=xRlMGUY+iKOjvNQ`vlE^DXt%|SJR{ny`9|-Hi zW1bt+|JH0z^=BFV-`%R_|J~W$e7OIAFHZ}Fui8(W!jtGVKpP_eu6_bo$3|0wissis z%r+FQlJwuIp?a5}PinQuvSQ{(VdOPf_w)atth^ z|NW;`{m7#6DRTnxIrQ&EQ-iD;zPVt7slfalCp?q;fxW8 zi(R~@ao}`G%*H@6$Xb%JQTR#cnET7RN8ZKJobVWGN{~pxQWAwR^BnKx>Dg(uw2T2`6H6FYwiD& z!~K_U4!vn`y*?K2|NhR?ekK0jR)6Oq|MUGkU2<&Zne3}P1kO<`G8#pKWD`a{)JuF^ z0|9Q(>y5)~k`FzfPkV(9(-q;kX#uU15l^Ab1570S2Cd*^|M zwj(~Ds#|9Y`P#t-t$K}!m#4DcwOPtoZ%U;3W<@#VL`)3}CFk`}Ttf}aG{hhx;nIBk2eq_;!@2o*UZ76C z^TY%f1t3rUb0$mmaEZz;X&jJ%`;hPeEwE=WR<$>9ypZmsKE_a$`EYXh`f&gB@a*mW zo5PQdBW~Cx_5kqSV2*bmKCX33zt_o-Mf_3%hNbP!myTDJTVcpQ#o-Er&g14`<8WY& zr)NL*KSXiu?SaXH9XZaKhR@}83&Z_C?soD-dY0uZP( zVaVk;WJqXZeo01o6wQfQV8e)!Y?AVPJRviX`^b4Yk;OSQcV5b@aW1JDBT33;z%Z$r z6FOsL$XHA+SdtMcbX095G78vD3GA_|8qGIMCegKB#cBxE*MMX zBFQWyL~{U9p~41YLl=5P00OvoN~!EpOXm{IJ^2Ll>|mYnsHFa zlJk>1&cZ2si6cKRW_=rH{bkN%VJbIzJZ5r;))b<0Xaiu&U4)nPkkoG zLU<~V+l-Txxk{B)UK@DA9oTd1IIt;^83}*Ya|!#Me9uNwK2k?O7>`@_YZzxK@8u$l z$7is2yMEE0n@t#=UG5Ot(?lVR$LpTNiC_#!WNBsc1XV3S5(AdeP$F7V07*1A$V{Hl z1+9E6$Ol|;pq>4==Jl+f>l=ipB+X%(1*diQEwP$Pb4U*3 z0Mdgy&U+5a(+psPuvp|NzBwGa^KBo8(KN^3@#0>C3jBl~+O z2>X<9;Pzs0)sr`h3dNnxL@b_{bzI0N+x=b3-$K< zOeMk4yedzDkW6VJ3r{ryqd6Qq!1>0E1;D$q6jt$uq@_S#aZLt@h=eCOC~eQ}Ou^?G zTgNS3a=4IE&_ByP?3*8%BJn*i`|IK&@o5Yj4$fD`NDz)jER~d9%QsR4yi@6|tF%<| zI4IpxUmxHMmqk#=Ih02ilKW67YD+FdNrtJMvdX+=)USn2feOdY3 zBZumM9L~))1#R1TDZ4hD_F0;<4cM;|eM1dF_$yXd8w&_aX=@?XhRkouyPT-1Iz)+- z&i*53>3l#Q|8{m3(zP@-- zTxc~BS2BOX!~i|oHG{~Qr+Q4g*FA7$fs-1o_VJL?~50|J$vSX)%OiFnTi)L zKCeqm$U@*Z`~CdIi}m8TIn)yP%5z+k*;{70B_^Xa&j&bVteEcleCbviXH0+mea)GM z>I=@AhHLd%Gs=r4v(`FUC)Ea^aT-PZk|aDJ2gg6bY5j&xc{;CjqjgmGrj??CTrvqC z%E0HcO+SlB#*=f+x~;YVHR2ir9VEG49fqw@6Mj`B7s{vgnI&s>b~c)^HPgVE zpX6t`pksC>n9t)toMk+t(Ql^2U!He58x1Sy!}j#!x@mXa%IVe!)GAQP$^cLqlqnC` zU!HfuAYz>j{exf9Ff%`;%oJS%%!b5_e8$o=49dA0B}DdbYfj)>C5siBx7*6=0@nA0 z2W`a@9#m%=#x^K^tF140#9gcmIvbx!ESyO((OM$d!`UTIFQhajSavp?XH4Ar8o3U5 zuvQdgZ`ky`Tqz2yrGOCPuG+2%uDNR4f5H`Z8O8y>9FXlP9uZHTynXlnaPZ^_Ib|6T zGm7#k@bgu|rYX7LO6pf?!wS9MKR(g|;f7L*Yr!8>Afr6?QThtAxg@-Iua@`7z91s^ zCrXTx;#b0j2tg}lgFhuho=vo(3*b|}Xx{ZLkSp2$Ao-ybZc_{k2eGb_)AU zZS*{i2Bcx5H0qIT3s1@SY|EiZVt)3$r{1O$FyW^mFopxNj~aT^k4U0Wv`bghv6HghFBoZC>)!0izly zGEyEzL+W2R#gtp-cVwYMp{;iPO($iuP!9Y@Z1Z(E4YPs#;9zuc&?`cKaOCk5VG=8& z*1<}_j(6Qvzb9sn^dToR8f$%DF^Ek_zVJf zf)%jK0wvv+9Nb-tIVu3?w|?1zFFd_g^Gywxn`~T*gmf*$B@|Ti3g;O&vZL9ZU;>i> z>jteUHTxEp=YcVdK+J=5NPW3sKg~Vz^Q15sZxFQ4O1e@h2u7roXUs?IW@(QDXHwYk zfIgvVn28Ks8Vcf4Xa`+8 z=D|9t_+DUhCu~#YhEaf~tqf=_Q7s zZb4J1Nm&5qd$gG8SE&23bl^&3QgmT?sn}N-J+s+=R88qUUk+w(_VLe*@%V1cACe8$2N6RegaV_&urDY0cQi1PgIaMVI0a;2q>NNoCKU)(zrAlt6Cl_9~1y@ z#O2-y#v+vDCB- z=ej)jC{YPSuq3Rmfzu^nJWiPq=u6Dw(%y2qB<6%+frO>gPzn!DY&XuelqXI=*_6kEWsW|8yInkt;SW+fiJbz`5*j5Px8k&*{VK1NrVZA$L*#V{ z`GpUaqFcWy&e?i&X`kr5~_*JMShAa|D%k^;s z#~~coKc&%t{B-gfxDTQjK{>%| zw;Ry!3W3XlaX-tbKf$Tdw_ZU@9|~Wx1#;;Ds)R>Qw4ii3#2O{*b`B%h+Vcx{V6Y4{RK@-m~5Vf~PLybq_0=OEY#YX&rxQS)!5 z@gw`ipZZX$pi|q^*R-RnL`|nqF?l{u85gizXZ;RZt!yQ)HD#y3JA&{j&ro@sNV?%7 z^mq_RA`=@q7{3IP%TH=_f%-m6G8UBZEs8z{WYcu57JI2{TJ#94ORZNm?JM+}70uhd z9xoi6rVdT5D`pi(qps7V>D&5#ln$`>J@2>$FVQGHx}r0gX!{ zkcFsD2=O9S9%ZHhC2v^1)mjfx!4+JPDBL1(Z6f@yB+#+QBzM1|kOeDMN*F_C6JV@o zx&AAS*)$waG8C*wBkc(NAFs(jbA%etIy8Ax)w>5As|K=__34tgn#YlIjQy4D)xg#0 zb42SNg=Wq4bqvEPc@2ycc4TpY`a%Hn#-0||C2fY`g9^Lo)12f)-4zLA+f=uZf9nzr zpbFY)?IIzWVO8q5%o>p_rLh>XRA?%r#9K?_ddvgDV}%7g3%ph~3wyB+Vf$ol-KI_F zDz;=8A&Do0ACQCP-|(7aq)@M&8&cEqWq6_GOb~W2w>Qqa_N@lpcN#J!QvSk+cp}Li zSzdr@yCjKMR=ddJUMshL3hZuAM>x{`)>%=_7k4Cy)~G+G*#vKv!og&&yXx7el56g0 z_Yl+R_nM_v>UqG=<q4Qa;>HWvtxR}V#{fiEdTwi2dOI$|X#amU+l ztns{)MYoKw@zcrcTkv9EW4Jf~=%33$ZdtM<0sDfKblDhS!@c-vj(%Z-t^Rkr9(f7Z z8-*eo!f+ABLeem^B?%b&I#Mzqk<2cf)CDh92-ZrJ0sWukF;M9jvcSUg|NQs=Rz@jJ z$h3SZ87wK9d(Bd3mnex-Y4;SvFkz-07)Mk08`Vv`3IG);*Ffgr73&7Ztu1p1;z8of4uXu)bU9cz-7 z@+p<)eiY3me=Cf_->8O^R5@aoL`oL{O#_=j9fd4XH~a`s!*LiFd+irKgjK3+z_Q^` zkRl2HETpiY&(5-Ga#qmNvtRfS?yeLv#_Ift<#GmoBUyznscD_JDA=iHm7C^-zNGr9qPJ4y^$TgOD(WhAc-@h*N^h7Op6X>?RXb=m)>PdEd<}}K zYe2fPn(8h8IEbkpbyDPSxD{1&ys=($xB84Xt0q%xW81yXpoQ+1k`LQ)4vU-E20omW;A? zg>6~EDZ2m^Hu+TYOKCMfhG)KT4N8)uwHk+J-$NSXu7Q6aRr1zHJvYH`l!#1uG0{biUjlxS^7he{hL?jhba z-^(!Ijfz}hZUj?oCltiUqbk4vtPABQXxuBE;%=vHPR8xj;tdAegM_OLCCIME1Xb~6 zksXYL%3rg$ySu$3H+dfKH@sFetoX(9>W8(|CclIZWk1QYIuFny{2PMubDYqYE2E zl=Z@D|4g{Zh_bai@`p0ut6|7ZIiV`#29!-wE=dSEdTHJ70tcI;mqn#FxP+B+-eUnb z5Wpc=IewD0-nx`Id$MW|zE+)apjDVp@kZ~ys`~|at^z|d;g=Ae2NtQ2jj>Om04=b3 zrCA+GF+hP{=??U%DMo*wW7tHsupJNuBVZd^01Rg=ntKJc(Gh)81nw=5*#`NMi|jo& z9%D#)zYkG50C50)mE^GkEVkUq9Q`0b!i`O1Mv^TU_yJTO8a2ykgX{x~mVl)aNKA!L=EXd(DJY02jx?M+qM8_7T zB;VIB;SJH@l7g3c8ph*O)65aXL{+a3KQTWCP*g2!>r|;T-?MZoicgY`La2aOASv1( z@FV1*(Ix>1s$D5Uh}NwQ$WdJTh-j_0i*2Jg;>u}pp=GB_?1fI4R> zwY+?mLuuKht~1EO>B;<6Kcc!OOjoKRa0NlhE{7%(!3!m{I-AN4Ldfqnn#dq}W;G6x zquN)-731ZlqO!bv+n^7(huU`O1CbWQwpP50XBX&G)oOA{-`n!Gy&Xp)fTj@cLVbxv z-xD)mCwOoJKBZyoDKBPKXgc?TSa?|16ZvQqesa3n^Vc*5-hOewGocnSEa?I1Y;{US zSP+ZN^4_;M9t5A)))+#`WIjy8Krcc`1MJdrDVG-`7Xv9GmBb3Z)+nbQfKaV0ENXqDN9jo#WvcY5M;1pvn zXhuBa5lf4G=@JHJ>j8Po73D&$Q$alrL!~ENH1Wsp7qg<_;_(gD5YJmDGx7XC*HSvY zia1ZMM#0tEZj@z#Tbc0(d%q-b?e? z(rQ*mp*UINNCF(WBJ*zD>E4V`G9PwGIXzlUY@SVZdW;uHm4F+A1@KJ&slVKWy_9Xp z73ks{H=8F+(LYoLbvk72rIyTJYIM9y4?w0W-dxEgVqVK1SEt;+VA<8dgr+ZHZmzzM zxPO6vV;{&?;c(2~YCKwSM3LeOcal}hX2v8ZcU58JuF5=Iec6q|G2AG_;|dyOfhVV? z_*l9A_YA+hp-C|%En^Uk_XnTh>&3Nwi2}>q2r z1OZ#+ivPcqDFL+&8%F1VgkYnhM<5x!C~jt|&>nIN$Wyqy+NG%M)3mn0<5?Jl)D8Uc zw73llvyAtvRDoWpz@YE#c-vTPX73(#a_h#kX<`^s3(-CKUZMzh9(p$*d={!KsI47Z zh2IbjR}&3UTiR9HQCZ%ap<0QcEQIq8sma&dnMZM$9TW>T<3tKaA&s;yJK=$$pXI_z ztI{Q}OL3EB=~Col(0}1M{(h1&G2u}#AlnWS(H~fbI&S&nN65uon4l{D^9r<=ETVJ6 z7{PVpGOyH^??eu6E>@R)dFrCQyC@1HDB?hyA(SIJ|L(`2S+*HV!L|ubKZOjYQ~=wStbNc|;S?DFlyF7iedq}#DTLp z#4E^XnyHB63tG?~v{pL-sJsB9Q8;3F)99E7jS61Uc{dutML8sF^3M~7Mt}~g_d1f8 zbhP~y`+WoMs`5jj`kB%=9D$!n;U-Omi8+90&>LOIYy%hK)paW#y)%8!9~1H&XjanL z+zR(uWLj!Lf1o?{xm*;!xbTj202QFXGAyLBBm?4-WW%T!rLYE!mY2~n_}jzE#yJW& z!{BDh$wsj03~91)QU2O7QT~;l8^r%9vVgvQ_x|wI%RXg``&bhHZF_HLubTh)>C=b! zZ};-_JxJaoiJ+yW0$;V^T)X+1Bmp`tfXW=HxDG)&Z`C#D&Vr=7NDy;;VLEbCtQ0HKT8F2%h?Z;&yQ;pUkq!oAX9hqS}*|)R@lzR?_NTZqYCmsX?q8oAR?XS>p9O? zZxGwssJEkU)ycNJ6O~H})XtwjcUm`K$)IU%xz67q1B;q~{?=1()7$j=3(&7Gc|bTc zi~M$!0Y@TJ=PbP!eWd4WQT}!-W3N+55?qRkGMAH7>?W{Rf^R>{ z*p6$m{}p!-RE{Hvz~s2(P$z&490lQ zk(5UagHQ;8wx;v*^CAMeD>-d&UXolm>2%0P9GPYYg;fs_6)e--ur4Mh)oX;r8E7m;4Mt!UZ`+^`E3# zy(f--tKBy+7I+XCRhvBSP*(`Xab(>#99aYsE2`I)HCj^aA{nS(+?Yb6wDKY2g4r!P zPkNe?Y6Qbq+*elXh+pM5(@_Wk^SwL~wCe!Wl`piRyHi-j61h5VZ;qX%e{@>MDV-VrfQRND(EMX?xqByJ~ z6)6<>2EC^3NS)-0WN9AzWN#b(45z2yErmTAN?e=rpLCWR1gv@%3$d*a#b+JlBXAO2xJtBz-r5t%Cs`b%399SY z9kc$U?$RE*tKZ);y;GGuz^>J6+o9x5AB%U3CDI?m+jzH@)5JP%ZTA5$Y$8~kRy8w_ ze{_@~K>$7T| zpJi#viiG^cCH@HpHdm>F2fh}`1uhGlwms}SjQ<-x`iB3H;Qn4o7p@6{tbM%5*5df> z+a8F9p8Ky{Z2cbDA;;hLc3^0Fu(0A+>dz|pDwMt+rq&{|kY08A+|k^*g=z9t=Ew@u z;g09R>Ov_7s1L>slh+F}D0eCY3_;#nOKWO-><)Hj91R7R1K0yyKXK{Q7-jlrJ^qi1aMb$zUn5{G$H)MeGNpnFd6IoRT zB)O74E#nTnFWc)%cMfDV_%$ug8QB__#h5n1jl!x%I&Ze|noce!_6yl+<`w&KB4okA zj8^Y4vB}Sw3Oi1&zJN3r6Z;s2hiL;#tsiANs0c5=_O6?Fbsl8}a$AFtRhfQ$7G(lv zuEeDl6~8M+W$4yrQs&z|gZlLu6poZK#At;tkq@C7NrARn*VjzzyCcvIJm$6y^}}IYg(jai~VBsqi1xiv~Eq^vTf&U z&AOgPeh!=<#Y0$MO(MBK;Be!Ok8T-gBBM|{gTAf4PP%}QI z7qr4#a8SWexyxJYk?X$pB))&H{oEG+Kg)_VE6#)4z%u^-Xz!8B{~w7x;hz8B#q%5E z|9`=0t3(+O&1FUH zlj4nfXR&s%UeOmlq3`;3yXI0Fp9dC ziZlRO@zRPxbV+W-W+E6h?X}gJRlC!*voVbyDosWxZMVIZ1Pb&}#Eb9?g}U*5*B}Sc zOn#(dXFKU#Lgqc4(GX8(L>8d&)Kqg&A;g_gH=9+u6*c69EDNLxdtP18dXC}OAnX9y zD(gN=lSSYbOK00I4ICQ#0a$d&K!n$86S_OHomfx0Y#_#yxn@`mdLWR8DV@0iK5o*j zu5hy=b6X%41v|7bZknmLB)r-ZGLCzQ@<3t>X_ZIM^sEg}IdCHi`W7udKxj0qNIp*B z4&yE;7Ge;x6a&L zx{C3Ueks*fg3zwT)@ejY=yzc&|EV!O-UvI^Ez;3kUJXqIXa~_}@EIJR; zqE*)sK}~!)Z*_W?#isgorA=?e^~JglFTOk#o#T{`*22r+1OpJ)@lUIJFzol#WBY3#4~8I*#sM2S#KdM$~QJaGON{Rq-!h zhD*9`Im&+dGFno&q37hwm;d2Yx_@xEtT1+J;2$hHdf4A|Upwggn`%&tyqZe5vx|0= z&)v(*unKwwg^`d=!)lJH{IZ7Xy7#)v>v)H+s6Gw^?~134=FRlqgB3MzJ=!v*roqsR z?zgHB4O9$C1eWV-4i>6A4pv?5>nJpAb;c54_sWi2R(5nc>945EC6|@`H)XvZ8eie! zY}w3kQM~bmu;shEUITseyGO$8nBQa*Vi_)V{gapYFKbwxl+;eME6VbYL-iU?-96X~ z`FjOpASa`;#pkW}Zehrx>=itI^}=n!StB|TtS+LnzdCffsdbjlplQmOq?)(dl9T6B z7xVL3-d=l7=66%hO9;QNy_p>&RG0G7SYEpiGPB;Ob71o{y*l%j!vqzt~u*GF8=tio-Ra8QYuru{ksnCvKs{oxC`treih#^JaDs1b+HH0BNdT zWVK6!8B4QvHX=W$1Wq@_@aqlLHE<3q8dpJ6r+RepTvqWqbnR*xS3wY~x!9TXlGng- zoh$a0^%cv=xVHNfw&t~Sb*G;AU+1}z|BqI1-NgTAclXKRS@%D_%Lq|+>jvd=Q#l1#x2&TALRxJ@A9eviC_K?-8S|r!>v4@%6 zZ2d!mIGxX~j|(hY|GQ6o|Gy`X9^L2vx|8SX%Ae3q_CYaS;y8xD%9S{)~?)mF~Z|}+d`oD|E+yCW=q87v!(QR2k7RrkAPAG>r zcKttugz#Vm$6>Vt)7$qOmTBIwcI*odE64)4IN!}Hrh&W!kn+c?Lj9`B^yH4bl>ZY> zR=7KHm5F|Ka1ky?g$D7thryp-g-B0gLa1>zTrqm4Ri?k&Mfl zl{^`jv*G0)1fRuit(dT$SlN3YIRtwXzDC>9GjPZuf>?=d>!>zXGN(izo$vbbU9o*61p_!h|?5x+f)mF_wfXWIx zB8T4AH4< z!GtW@Z&{kv5G!(BM7{dg*acntNBn(Fy8q|uBW`y7=T*Ey4ESaG-`#`m{mURQ>8oAwowe&!E{%SGTc=mh>3fnxboySj{Y*Ryc@!OlYMGf0p!{I zIkL^x$@|T!OqJ;`P%UFvNJeWH&-=}mGpD9?Y+Uis*dH4BYfsnv=a!$_0o#NKL695Jikl+@76GPhHkw-g!#!TaqAVye}%C1Rh?aC zIXh?R2(>y9AsYmtk3 zkHtq?r$@)HOeHZFc*6?mHS#>C*-U>`ejeh5xIO0EXCU5FC$T6!Sxn?tE##Ng%r6`F5SC?gDNXrT3iT(vz?X(f!uEpN}v*J~yCvO96(9ARI3+TK66^xTO5D?^zjXTW9Wzuo;q|NM7& zpa1u69*_U45$&<~)U01>hyD;ptgmX}-$VbbdePGSq@m*r*HJpFzN51&&lujgd!b%J z)lH2qI`Z2*5W#H$sJV%q;n*&iU^F->e|EJ+!pomy@K^W z^tp}wpUUiX*FFDr@;~oC+26nS|GSgN(*G{f>Uwzyy3hZ7v*$MW|1uxI zCH((j_wjCr{~zAxf4-Ba-~VrR9p}H*Dq0O)2EOrBT!`bi1*Knn|cl&OD>;?{awkO>J;(Pvgt910M_`c1$j&I*(ISabI zi4Qk&r(Dj(b-lS}cjfXlyLEr3PV-BAm6l<)zzb@%Y10!a(w0J8>mzAl z{K^XziI-u@rS1KP4>DTYhYvqn_%vo87!e8SLzY@4YF6cR3>FU#Us(Z(>vhBr=ys#7 zh>iNXo$2~E|Hr#S{l`5wI{#g=y2-{guga9RP07bJXMtnjvirXSU;p=L_i*q2{_jp6 z+x~Njou(RMC+^o^n`|j2qo%`gnI4(xAkU8-BhDZ=c?$e{2+|+ovauU*RG!Q_dvBPU zH5-?+87oZMU;EA4vbeG-kr#ka-AXTmE@nzj*B>wxw9~;p*(Tfz0jVzYoTYEO$yx#* zZ&xf`hLcoS1`xt8>q~gy7H9Mm*e=>Zu+6)>=<_Tqx)|tp*Tw)P73tLCT07CI;zMcj zglqi;WwWw4VK?_NS+Ub?mD0w^U1PtwhBp59EGk`N|4*hYpJnGoS+mvJShD}`9_;!4 zACDd%9^CK$ckyhHW7;&VE;wmQf}=`4PFX?D+AIf?>WYpp=sDv_Bu&qGTY()1;Zv69 zM4T;9NtPAoJ8(*EvP%Y$iLCduNTUr>Fq!{$tE$-~`@~XQvj1y4A&^P8ETE=9iBzm6 zc~-C_N?x3NIBCk7MH}RKIh&OQ`TO$|l4doJlJl$?!oP@ql$`y$9>Txs#q@kA{!u^q zWid2>&gl4}t;i(H8IK+&{9_e8OwQ;<^e|~=RrK)xL>uJqw9d+wlUFZZ@+grW3sI7# zj1I9}UHbmXfnMdHCkl|0Wl#E?CZY$j3Ts8di|3 zAb%}!!B=^Eehw~WSeuN6gf(u;nkP{Nd%Q1%^@XrTPXt)itLo_HRMp}+r@FN2EHGfRq%%9Ie9K{ ziaMRn$QJjxEyBIVe!v2$*{r-|oQOhVN}6dbny$)%6%8kp^KzWgMs2m= zmj+7;D&4QnYYO5;U-zK^yU~!7IF=Z7Cba~=yiZte zd5<>8qvVs2p&lonqUfZdbt8H=smmF;lvm-8lig%@2S|M?5Y=4uR1QeSZ3bfvr#a-O zQu$yocW-V1|8ZH2S=I1idCBU_jC~x+fbFrE0da}${=+c{KrsRwZYSj3l+|n^M8J93 zk}17pqDMGI04KvZ&rCT`!eBM!f^v}xlb33rT})tJ zmlee$#)y1h^5$LnT&b-P?8b(4TmyC=MZgoD!z9T5t=1keCGr=X<8pJ#it`?Jkrho{ z4q@ef0M@ZFHt^zR%5blZam^^6BsQCbyekQ33|P3{mBhwaR%k5ceiOtQKyLv;Qfs#z$Q)%%|l?Oo7X z)fqN5W5XF0L>Q|2Lxp%uwF@Zj{xiW0a4GWS_juS3JpJeY`hx90SDNqo2RzLTfibQU z^u>CX>HnT|@_+3g?A`l+-^t_Ip^s@p^YUC73W%d(mZq#2k%3^IgJ2O@RhNqIZ2tKf z*-Q2hlihfi{E6%(kM_i$o589jACWvO+E3P3IVHvCF{_(wk_p}|--gRsH+l%$n$IzY za|lP z=|hQKLfAShrLMxP6OeXcMUan_lXF&xV~@Z=oI0E%26<7AFIX**i#EvO6P1x>)yl(j z6k{|!V^0nb4*ZnQy;NS<-vCMRrm z$!bAa-qch`)4Zv{aBnMRldM2}ZGb{QfLt`oij)sI0NI97uvh>orFkVwgcftf zU%B63IJsPADN70E0y9oZI0%s$tq|`Y3(>ZiHWY!Pk|wp621@3lb2b6sXspPPvI2W_ z#-uXUQn`tvGStK-UrorY&6}*snd%3GZej%-1X#^dvX!=QFc;u+T2%r;**y$`w7nzy zf%R3}!0=(sk+M!%jV-<@A(+VqiA8^F8PJ^)IOwTGxkJ7eKXGa0xf@{08cx9qF87lC z-DI~1qS9W*aazygx-E8OM69}IM&$j^(3Tn6 z5g7;xV}EZDZHV&%nrFb@%FvagPxjvK{=vTdAU9Q2j>yTce|FZB`H&S5f0>lLac7S%mi*1%_(QQnefNZRu&Arii*|#zX3iicLqZbCvRk@QRHA_KA$7b8}T2oE?M8Hr-j# z;9us7o9t+dci^U&w;=f-eHK*TERLePQbH6roaDYAh4P}1)D8|9Qt!N`XIu`owAA5h zJVp`5mt)!mM$f>OBdx2lY9VTnMf6Q>(t;CIE+-n2#pqdGmo+I{xOS_-WDTYy@*eT0 zWjn1zU*&(6al%Pj%CiqkRs z@+IMyV`VFz8sU` zQ&uwz@tX05@8GQem~lpOMlWH2i#E?A+j9gYBA|~LGC_*EWjl_%C}=e)>yNZfL0Zs? zIL)Z0VjNx_i|Tv(PmK&K=VX;XxYG*jf)H?K=m4x5hbMlGiATD;(H~eQ4SW#z&e2) zkV#I@Wjm7HOVCXT7d*1!;B+mZ5za^@B?e6WojZ{zGXD!vNJG^l|V0!B0~DpT5Mefj&YN=KxKNy#geSq z7(~^qP^(YbSg6g8&?*Cx&*)st1iW~L8k9`hMr<-tDDQDxi+V&W_1Y_c3|Z|4!%JEZ z>$VtTL|E?Kb-M9KhCqnHa_S&3fxMQ&z9xPAe0D(!U8_4iiQ;;O0F^}p#B88nkFDu# zc^X|XaT0Y%E*`wLqWO9ZKpDTBoV+F~!3iDlaooh!t(M@J7GmrsWm}+sm?-}utL640+7VL|hzzS4ECz|#80GBn zMHg7mi3Ee92poTSRTdyj39RI$71vFoqo-{1g3X=ZWBS}LvY~VVKguh^uHMB9MU044 znK)Fly5Wi1#BoXyY86IfEsOIuB?Qbxbqi2Ht_#&q%~mBl7UPjbY7)b?lEN^+ehPAB zHX)}j(c~OG)nZ?f`bA`WP(-zf?YQ(nQB9u*)&BX9DU(r=^;}eN6RWbxF4U938)sO@W=R7Rx6jIC3yz?w3V7CkTmBbS4(UZ{eCzoctr4KX^Asa)>A3 z=10O|-XPXGKye*z*D*Ran6tojuk5v@c^AA{lsd6xd_tNsXSKW{5r_~%CzGtmnz?Ec zbeA$@wKnq8*9k`Awg#-B=bbwQ^IhJ^kI4IhS9t)07?;&t@B#j6am%wzoVZvT=h<0J z>v??E^7$FuEJzgKdS)-u2Nrdl$cjdcj5u{pI{IKO*=O9EHCCW?J%i}DE;%>!SI5Hc zT$Mn;U0Y1luq|6zn*rkjHM`N{G8N279tN9M##_RzfRb@uwy9bZf_vBOqd=^x*(Cr! zF4Op38GkSK52+KDg4$WzkfLl9bll06HL3D*q2R-IyF&%CE;*+ zvSt+I7NvN%`ep!ca#FOjGgiYiVQ|j`y4~U&HN6ZWi(MWX1K~*Qe>@W#zFf#yUn%r#q3AuiVk6d#e@D1!6bl4E-_t;_a&Dv$5pW*|@BW}0Y@jaDw| z?EZB3>25^GO3X_Z3$QO(JSv3%CE|dXt7Ec#!?+>`TZ3h(-t=%R^*O|_;M?HmKZv2k zAM{!G{?|Bvf0(koVl_{iYROgblKbDshx?uUKYP3P_8)ihY>>CoumW5eTRIoq$?`VVkBP-JKBPS>5nkG6k zd>E;>TEi$=VlZyTSNvr5S4gFo#hdynJY0dYeG{LS3fC4bEtb*(4hoUw*_ zvBiwdj}CPuN-iuE9`(QCEv4HfT2=Y{Z-M(m)3FEo+M0}>5eQeQKEb`JDImVvxiNg0 z{j|~`66@yuK|kGWq5+N{MB6h~pTia5$2x16jktj?4L268kY0c)IHv`;LuZ?~+9Je? zi4dC*aE2*#?l)yIEV!(cGck0fx6(c3jjCmr4$TlBb2JA=uLl(-O%#+?VCdIxw?e*r zv9DLHfI8-i_A~jVEgLAR&K4tLCi5y7sf&z(H&|P>QkM+gsbn4U6(=kM^@SCxN#4uY zpsfcAP#y{vUtHw`_yj4$LUVX!p zZ+OBl$D?o30ogJ^0$f@qGA_WjSHz#XSfB~@&WkSVaUQ%U?44mqbwtA7`m}uf4UaYH z%|Lf27&v<2=RtS=ZLd7-5h*v;E-0YitcYhU>&8;y9;5d6F0s$8ReSN2$F!>aTsGlR zkojxE1s#Tv_;H2NMfTHt;p$ysu<<-}C)7fe)*+U&Y;Hs=FM?}$`*7_)dWiQ8kAYn{ z9QiKD=V52lF-*h$KHY?tDVNZ4-@Lz4$B#~WN*<_W8IbrsGU z>(aNsqJKSXAarkb`k@5M+Q|3omU|&CtY!5w$TIt=yNuB$#3Fh(oZ{#i+C8Zf#`wwV zM&BeL-5t5IeV~XVUkFhkfe1TtI+&T$4YMvK<4{u%Zo91qI^3e9jF!_^rkjn{f>4ew zg57op6mEhCWFSK;4pypVM|2#JEkJBw#dREPuLQ;l`8Xg06TWgaB&=YNgVl-ArR+08 zt4fSc;9IxZ{?QcEzh^v`j+Td#*fgOSM zWoTr#&LzWAB?ccIV>((DyiIx%~@tldg>xe?xo}4g3WJ&q19w^!RBP3jH7M~sDihg z2f5@>{lASN2($lidw=LLSi-=qKV@Lat|vOwEQP(QlBdBcnSeB;;@E35oWqBkd|Y@c4S6YN!nxmbAR zS@Fv5s2dPtvx2a)n?Jke9^!qfOkFD(Xc-D&&JsRAQ5MR|<)^o=A@s*k+JX(N&b=R;vu5yf-{k>L57!1z>CxPTx65mUm~b4& zH<*rp$%-^9&cEucEOREqE33J-pcjz}wQR4cwQ=SxMh(N+7tHfWdeX};q(px__Rxh* zX!zDjJGm-a>mqg2Ar+nKSq`f$z8;E*vx1BR5pd& zkBh9cFW3Wwo5JvN?~FB64c2j)p2!^mW0C*9X0Xa+P}SLGma}u){L-?9)uxu>-K*p0 zZlU8>FU*U&Espq4oXJFNVlN(%&9~z1Q8C~2qabv?-*CHA4h`Pu$~Ko&_IxrVtLu0s zT&-69v)cH)oNiZ6+Pb!?a!?x5T*@CdU4CeJ1GL%1w_u}dqHE2vVnlYeE6!%q@Yd8BKpI@;LGEFe9# zj7hrbIxmePmNFp>5KQ@UnN&8GcUSLuQ`cH89gpXeVf80y zfZORy5AHVD#ph$Eq|T=Pv?D@*qlfJ+(u1pGv%D(0wi)$8Z0&+ekPQiz;zwxQ!Mf>k|Acve0HAY_7 zd%XY>;P{u;@N|(?|(db_uARC~E> z*Z`HEW16P5fcxXyciW3l7Ii=(jm4JmtSfq|_U#}fO{qgY+it#RObNFEr(fED>)Id$ z(4%Ug*LJ8AAN`iII9DGR^>v{hBtXxF<-?cUHeRYk%rCMkzJxN@MsJy##%>r?F0nXK zJiA7ui)*9kohLLlIevOJWX*WUc@84JjFfEtOezSb>ejnp^Ci{d3l^?ArYkfWk7+E5 zIS_jK?Y&~frM=XV3(9qF@elB#_^vKzc908#@uhSoJMTS4Dd7!mTArXf<3?NoaTBb- zM{Gnwj1;W>7x!KRReY-pd67jh8j0Ef)t!Yn8nRD~O&3ttyeSKDtamnJ{SkCIWN&wO z*Qe7ho#~-laPo#U0lYi%itmbwahlZ@ikLtE#xyV+ud&?fZgNZ4=ePepL)SE-)!y#H zvA%6W3iL-R1~ynON_gd$$rLf+Q=0*etskGQnZU-=k_=85ts!}x z73~Ff)M^g;92xgSI(6f;8+4!uB8|v{tLHD@zWeaqtJj#x=1aFAIQ>1i`rFHYE-Z%p z5+ELRU|OZkx04@#di(t4vjdPLX1q==Z^iEGTK(zJsIM)nv4-)6#Q$~%4IBMJ{_DTwIDCg0Ihv9H8d|3} zfFhr=hU^7Fdi3gtc){i?0ql}y0hTm~XJ?WzIM%9)Tu!}fA02hJe`#(dHneC!t%YSEAmOqJ+cGh`dt`rzlvXe8P@&9`{_+P6z_~K z()?&C)w>k~wU$zTl^Q`2%UyRtvBOuMnn$NOU$L{&jZBkE18Ve%n&5ACPYn>vzLAf5xz!9TM(QASZoj+C< zh4Ef1AfVmCjiN&3*6tPe_JfMHb>-?9x8z0QwO_!y*jm=Yx{|wdtGGJi>PpZa-U^!f zI@fw!9^K#n-ocZjZ^{O{lRH{iwpwTxbGeJp-;lh+Q$w_))bAj5CK z;5C5NZCeX28P4C5@#NR1#>%FX_RH(oPHLk)c{W*YLwPG^g+^6yD`tf~;BI18*b#{T zLuBLIlK&QE1Y9QnJ$d5i|2sUmm;dhO`6}|CGXLvX^xYoIz6DFYB|Mw-80uk|C(Y8= zwwmj2_7;rh`hmMPb0k`#;#zRm8@}niWfM3#0hSuT1kc1Er1eeKwdrv(7k4L+Yp6uPre|PcR%>Tm;L)Uc!>4d2Zii2kECF8PYC0O_l zeLkZCO0K|emV{&c71&Ns#K>O$V{&(rtE=VktRRm8IrLxU8Hki4lPLvQT>8!gB9qL{ zE)aLcSDQ2^UBOi=AcHE%9>U$a+lW#fJ2q*l*15HN+j#a3B)9429g{6Q6Jq&*Y>o z#ml0*B$dceUwLeD&x0@*nU>+OSwAdXL9-8rA|Xk^lB~4}JgN zCyyW9$A7w$XG3mM=MCA)SwXh)vb<<3GFWsyI3NS-UTq+@kHKJjo9u#&xB;if2are8 z$*+MQAm}N69P~+kLs#Y-pbFmtA`jV4HjSg>S8{jZR>^EaWVAxr=vGsnG>~^(2EX1Q z|342er-;}@EqE$MTj-Yzk^EtYJYb(Pjv<>ROb4Sa_KPk8U2`xoyM<1d`TXq*Opyr5 z8;kRV979ZWA{QSyHc4rJR7q)}(ixkSwJzsoSxaE(miFx{g{HfI?%=rv{$KG*?E%-~ z|GS5~UH*S~fB$nQ&#%G%*SXi80TNy%5I z082~8p|L5w#N6;5Fy!d?mDZwaMypa(AD0;Ijt2(W(Zhs)tfGg>8NG-eCe2LzUDnxo^zi>g8|3e_ z&dQdPS1(@jD5>i5Uu@h&NtQA?#Nu`NuPC|X<1%H#ufDifc-CG2Z(kn0_~B(TOV?;) z@%rCC+?WPTu<+ zTZpWj^i9p!@O&Cqg5eDn=NtN`tfO|X?@4ONN2Mcql6=jBICk?V61EH2tJhWF7L zmnE$#`eU+_mMiC74ydqHY?W->zy?cBl~2>@3ZxAu;)lLTMFnw=3FI=PgevP4B14wQ zE>^4t4Tj95JEIqv(UNh{cAu(X(WgtgD^^U|HkQK`dd_mPl_e~ZY2C?bH~7oxb`-s{ zCazeBlJC?^uIbdUJgS-AL@Zl5b>3SGO{Rcv=EH?tabb;ZB4vpriZ(XLPbMRNY1995 zFWEo%?;S54FwK2(QhsX7${I%2%x;EcTCR2(7|s8h1VE?FTcDjJy4qNRC0C*z#a zLI@1HliOk*WeukB&Y3#Ee=mSZGU7xztIG>kKwutbN9s`c@^(B$;!fHcvgiT~+%kbT zK@);)!)CoD5iaC(Jl*lSutQigPPP+4DNQRWK5=r+8p4})e4+IQfY}bI5a)4OaL7hJ zWt!WFWr^OSAA!3mVgOvH$1-4)*l(+{5YrD>!=ky|l&E#yL8{0Fkmqvl1f~~8h(wl{ zHDc0FX+ANlU&&v1K+Hhs(f5Z6CEVk+@>#P8o$Cxrh0S#3vA|IHp8|5mJT4z6dGFSR+`b+|hgqSN207g7T{xOw2Y82v`Z& zH^eV>TM>OG$EpGHnY>_pTxZ~LPCk>DpXjX08TlN2Cb4*k{6BVoL4i{(ah#IR1h#FR z>s86DXAA-ypmfVp@>z;cAjt_B)+M)sf@!h~BLK|p+p@pP= zA%|==QipA|P63Ws{9>)%!g*wPmX~M485Qhe_@W%Q;8-Lu9?~jHX6c4&PZ&DCEIh7Q za=02^DX!Y*6$)$FFg39;nX?Ny633%d_gkWXU}VmwrX*s3sxswfO>5ypT5v!ZBB+J( zKh7<8N}}Ba&Rh^0dd5SVfQb<~!gXCvj3Jp676ca;r=B!LqCm_6mw;Nf1xfa$iU`6h zHeM#V>Os8Lz$#o}MZCX4>ljIjFF(88)|N$Tb!h7nHTZ&YgN#>JEg1V)UESp5r6x=X zUYrTC53|>?x)#Dw3yh#7j5eyYKaz^P^#RsQ`7o8fcB^{B+3m%6$>U%~P%G5Dln9l!Jv6q)?Q4S)d5h31TF8wXNrneETvJJWqv?;)?A8qm;dX zy?$Z)ex`DP$af{kz;}wrqA;=or&kko&=kvVr?aA@a zB-yFeV#{$|`wc>@T0`zD*PUp#rpzJ%SQurfJh?q8X!9hL7Z%RDz}?0QD@55b zot(VZ-rZz8W#fw_(_Dt3!{ev9L6Q=^5qHM45DWpVUL{|FJu$O(O^cC(wu_uP(WV8^ zObDH2l`$dJ=X0qs8#B&nJ(tn*Ac&OiXf+LfviW;U#tgq+~9G-Z>l5Hhp&Y=#t@JCOk`jAQ zPbT>?IRm?WHGYzX06&u~PHUhf7#eO!nKl@wuFy3*>?2R{@)RxMl$C6z)iz!re=#;| zBp;zQLuZE1FlKcpz<=)}^h!`sPsT zlk`<0$+KA|%`O~~&YEI|{e?i=v0M#Q zQ&8;1;{-YK*9#hzpz_Dkpviky+w#lRof>_fb6chChs<$oMd)}>QGPV0La;%pb*Gt+ z(n&qteY)$Qa=o!c1j9qw4a(CukOmO4TunLyf+EU@0yy zdlFm(!X>zOG$(8(1l4g_@TR74*8<*UoNVEi52G5mdHDe@Q)XF_@*xKx+dyo22Ro;; zTziT0gnU=lI2q)5B!tBFbn04IZgO&oAdzh{4%Pzl7Mc4! zH4=i@I4;;+Unk6HCH6VdDp(03xE)8S4*i4~1>Lf{jRrEZ^C>Iz%~xBc#!g0EvqMd| zJkVDL8vB|NR={S-zzaWZK_C#Ib6SDIS9S|y5pM5D{IR~iajU94Lj_1#r>suoB#EMj z@3LwH3t0$kLaar;*%e&>Bt)yv7i|6+#o<4rJUj;TC3M05nC0o1)+vl29XG7z4-*na zCyW`3S7Xe53pTF?*ftIYUf_e}0GL3S`DDpO$?w7V{r*{C{|`nKV5p>W6%L<+-OhqY zkKt#T|M%g+VaNXeKK|p~JP>L`M!Sm$p>>l@=onZI)6!BTGX}C@uDZ8m~IV&m& z!nT%Kj6v$c6{Y4pMp~*h+Fg-uM3w!BR+av_pX}`>dr@jkf<{C-N02TgvE|`Ou3;c! zpuDNJ{WxcrEZ4zJrlr6uFqY<`B*Z_W3pW2))+w0u<4f^}rb!fA*9=ii476w_qzM%m zFeYq9vwTDvT4%o`*;!4e+3bI2;&4<>8i9!pA_GPCo_@P)YRzcTviyJQmYpZ8)g@=* zg(;QN%Pjq$-B0N_nVHz(Y1yRX2>%X@hYGzH#n+%tGq2bP1qswxM|$=iBzuR+<6mJ< zU5o!)0@yljEav}@clVz7`5*T8A3wS0|9A0hXe&V+ToCcPL!U%uVV=OrQ3id*@S6~P z8-r|$B{IGMHil{EWmHYg#)h-3uFIOG0e%)@afCGVTyQt!XN2%69g%_H`Bj_eD$Dn) z$(yn{t{G=V6GfdcrR4q3(T3ig2Jk%=^<%uT8AK5pOplNwH@S?jTk%T8_ml^H6HuHG zWa5&9Z#4-9kcn}Fyer9+PPQ(d@|ZXCoNe!r(ygPGn$AC? z_Rd%yowZrs#90w-0HieIW6{4knK0V4^)>eE4T%v;W57o2&>O}SqX5&!I>XEWsu|Tg z@fWzh5#CTa@y}aY)1qOFCy{`ypkhQsXK_WUpChs-2>*Nvv2{>8pw+Ov5%S9U96i=; zkYL1sSDk4RZIGHx3;`DT4ty-@i@c;Mypqn(d}!JnZm33q7Dic-aWZ&YLWrH8KnF1p zBVV!F)}qM5;7k9atx@x#U4S;7`kQc`&f&JFB+bW?)NN6q+ZXy%BDbc5Ps_RyO=V>Y zs+erX1W(Bs*?i0JE*ha3v#^vtnGP5S$>6^YT2NWI01u9CFW~25UEpE7L6~f5s!Z^;f;t+vyeVfo zyyzD|omN#)Ru`a^UeuVEV+)b`tqN#lEcUO_m>OFrs7`oktVOkI)b?sX89=%Ds^c$j z==cJ>gg6?eW7V_rQzYTbIIogN22#UB^%))GzWHh*Hz~R3{|f|+w|_Z$Zc&WLuTq?Y ziYM4bDwZ-s4+czA2u3NQPvQfK3BSltK?=VCkoXmovsUnC*cMAvpxPF5cwtcH>VGSX%(^u;3!QyH^|RnYQI$jw`uZ>2@WnG3@vEdra)a&XNBOC|7wdd*jB;xLy$zl zL>YzrAjOuk<6S1ulCE1gyWB));YfaipQW&7=RO;l3D3ALArqgyALJQt20LW%kv8Kg zd*L1nz6*432Qqt*4=_wU{v8-u_tEyIaX zV>uJQ=VFq`CbknP1?c-PA@qpp4h|nCCIZKKjh2aGc{mjsfi;+tCy~HYEK-QG8jy@s z$^%S~wDf#sdMpFMinG*iviCT_%pB%4wdg!02I(4mv1}aL?(D=(&f{@bO<5iDHftCU zY{S66Z?F-c38MUa=uu*$<@B7Xsb%(ALN+$^t)VuKlO7#P*wR*NSR9TP++4$<@1`M9 zDb0dlAC{hKTgPR(?dMiOaBr8$sSV{9AethX0{}8MUeZy-e&a13ISDvPn4lZ*oj2q8oiZ(pTT|ATLv2j`OaaOim0v}IHu`<=| zb<4Fgvx<~bi~SdFxX5XC*sPbA&sYJw;Q zu*XVbyM&j(I&w4g;J$exU9li^J1TN~!!zTYF3pH{bfgv0uv&7C;0D?z=|rsLMh`3s zafuR@#L7_?eo{irxRe;F7H8C*Kx%0kCT*UhSSW*mD(yIce#<7TW(B%2i;Y*g&P!(s zX)+sj(z~C754_u@gUF@fqX(g@`TRlzsmx)D0abz5K*-m3a)eBW+@v=4xnQG2c_xOJuN;bEF za*%059E4TV5=W!Ax% z5YT*=%~;tsMktqFy!L|7R@9}6bR`ZC`!*Mn%U|wISnB(xsrH@gi;$K>e3+I+S*w+4 z%Hwh9ErV_dnOHB|I`EEz2d#~_5FE7$JGv`L|4_hOsZ8Yhfm)tJpu-l>*QP0@H{i;E z6B$tesuzrv~Q11A*DoqZ^`zFX7+Gpz7*NvCq_X>NVDKAC~B@M(c zme&#Z<$}#eMEq-BH#x`iuvSs26HErd6+6Me<~?_OWF~}1BXvY897nX;T!sh>g$;`~ zHloT8vOEfGU}{(;`c;|QMbzipKxueB9(V3)vvDokSH* zXkJZe;x+-tF_s?Jr8rJw<|eZFaC03P@-$_^7?Dl9-c{ceJVj6g@QgJp!K;OwKVt@9 zWXmCcmpPkJ^JA-iR3y!6(VUKCDKqIhlF$Zm$|REt)5c37|fFV>c=5jk7#!XRy%0;2eHJ`TgN@_~F+=R7bX*3?73FmEg38 zfV~a?v1pZnvZ~fw0U(1l)EQ2cJgTx2DCnjI!&6gJzAPH&P>(mXZWE|QV&^sET||5duO`7Veo9=q z&g7Iw^57NBTb%du2FGFC%T-Ues(AXEYNtg?rzI+<kSq5g zjlet4x#iZH7Gv4Ov`DQK$-~G=sVo`!yRs_t@_bHs1wxzcGjKzc%oToB;y-aCTZ^%^ z*lJCnhaXSi+jH|t-8UQ=DMU($PJAuPIDktt56+sXdzHBiP*tWvQaTFVfdVOnJXdEW ztdpE_{zhKTfhkA^*I&^Fsr(|!dS$g$UbjVT86n!MK0O->1YcylZh^i0rA^PJ@G1l9 z)0nf0qRL1)z_^}A@~TCr6M#~JOVt@iw*vNwjvHAE1#rRKhGk7^cFsPj`~X~Cz5pix zFoFlBteE4Envq%Cpvsti8s{y~E?E*G;3HMZ+$t@`UaFc!4+?Ep@-=(#LKa)r0c%sR zEHUt=8eDRmgRi{1t-~j zW&(FUdw=ni#{ehZ+cAF^J4urKyd#$0v-h;BeipyCH$~5mRJh&Ehnwo6O8Y@otTtWA z3f@QqA8A}s%ryzlS{ayfN<~*VyJR&}F^wUfdN$L`lso7nRu6yP+*XqXg(i2>qqMa` zR~Jq%Yff1#Z`GM~XA{fD?Up8&0eRCI=J4W%(V0EDx?gW^u31#MQw! z)0)hlY+Wa1=LMWv@VZMTR~8%Pj8^hzT^9c;&+td8qBzQ*T4ccAI@%)qg7`FIzUNh$ zf>q;~=6J^ytFscrU%CM>50Zaov#dd9Mfg2y8!*iI$j+u^dErQGT&J$ZxrjgUq*%$& z@%XMxd{Y}QO*t@gO{pU)G+BKRB@=pQ)bPYk%N}Z(uBGM$9U)jqmR3B z0ukU3U=S*JrvQNgt+6GClY_(#rhb7lcA&AIjvb{b*^*XE+h8H(1?UjmdH7L>XK*2N zqP(Y72o#BF3D=~Fu^U3;2BRXCp(RaVp-#ANq#7=NFmO|CI@^?tZNFafAP&j-w zQIJhIPiND@I%j1w#j98{WA(Y(S{1P^kUlhR^%vfqx6sCao0K&HuW7a0xnaqTM0*?E zHeb}Sqfc%#cfttS&>72Q z)U!p`RicfJs1xKHtNUgxzoJpguW8cq>l(EDD&{P8(U3dN5p1T zGbdYMW@-o6sI3Np0csau+!gJ1wx!>el3Y;7Au~2wfD*ZY^b78tHAiwEkF7GXb;xk7 zO%2o2G5q_60vudWTujtlMXS`R#aN}VaHXju2$ktu%QE@rj)7J-o|^NMob{G=v?Oc?(&wP77$c5B@~v?#(6dEGx1Z%`L+gbqr83fMo8kGe&5h zUE(_U`|%M$|0oRo*fe5t!&G3G*;pE@143Ex)^Z~An^)#{W)`LrVH9apKo@i^gL`z$ z+*3C)PSyel>Q>Z(lcKv9_{?MzCQ~1lX+n;bVY8fJqSoNY$z-rY2Cs^&$!ISARNhzv z-2VoGpA3|#Cbs)SDHy)^AUKy}S1Ew1fORxYU9(Cwps)gGL7cB>gPbs?HX?Gy^73QU zF`e8X0lP`u45AI9?I!U}jMWWxOAzx=T7Z#-j6TYY5CM(C46dIf3X5=iDUJ?gMvH8s zB9cr0PRg|tEqZDFpbA-B&WXyP99A!~A|1(_*&k@7E;Ly~Q`%5vM_y|Fx z^0Qr08negnjtpQb4b+vnw2{R{&Pi7MOA7e5Ex#_7m|M93@2=B0E3(FR*&UIK>|=Ib zXXy|ejx5=`TeEl5n(=9wrz7$xB5;B7S0>Hk#gFe9wYE89_aweW1Ui(_`~}PD{6w7eQ$8XG z5s?m?`qiG0Pk0NaL{^hx{xoP4X4S?EF`8y>7i#tD_t1y09sA(^Su_5JviIi0H$T36 zd6G1rnpN6Z68~wxoB#RI!R~$hr#pE9W`K}pfoHJ8ay#)=4yxV0{2bDbNQfrt6v6dL zlb(TYead7~Mrl)pM;4wy?mFdyr%$IH53lIvuGTt#`G#!Hmm1a2*qOT~aB`U%uS$?!SxYn*0BUDa$J%o;TI4;4S{&4(4Vadsq`9 ziaiihSbrg-XJkv6iu89n*|s2L6CVhE!>>)9%}(0MB>O}L!$G(t;12OAtco5noZwS- zy-@!*$Um|o6$k1QR7MMu3?D}7tx9_h8eW>F#=HiCQBTSmgI(es|4t-|fNDj)w&Q2Z zEXH77NybVODo%@On=4Dkk~14(^l<186MO~hRLfA5<4DEm4Q+gO|UwG#~WO77Gn!|L)9j4E+2YEvU)hPkYJU<9K&KlINCX zaq>44XFK_8rY>LrPQVGm#1?fX!oot~SwDP8F0!J&sHB`3r|fHgp$~qObxEi+qDri`$zl@A9&EP-F2qi!nLKQqQ7do z`7OnOuQa}HJb*WSBkdMTq=}B%E%fl1y=$@9zTF2J`sEYaYh)n*7_0=Omzqa9%|h4M z29JqWFn`GL*5*wXn?jb!97Z#moq_R>3=+L67n8?kXG6EAwqvys+@kwRh4=MtLf<93 zxjpwR{sMU)VJc13a2@F|;9=Z#SZUGQPL85cayKIG&J3mW zx=V<&6E7QpcT6W6JAN?7U#AXQ2_caEvXRX8&Nz2(3!_e`m*xk3uFG9@80b$+PmxD zS_}F&Ha4p){jYu9asPPFHShm0K#c7tIPXsjwAB7**T4UJvU~9O{{HVS9?2GIksAG$ z6AnpvhHbH5dP>NENdue{gKa#k8VguEkWnP9U<2WdY)e>gW42wlL1g!`))CS@kK1)V zlOiipRy5>sN94CnZe_y@j^nvjx_l-sNPVhU!Mpo^uXt>f1UDXL& zr7NllV@0bH0FKLA#XF9{_{#!?={ow1u`Ep+kq;PKjmvyQ-aS9I$Ia14Deot}IyHn~ zlcv1m{6s!$70}b&r+cfTV>b5tef@X-TzCIRlL9M=$AYYY%ksbO@B8^*4-X&R=l{Et zC&0zQd0t*RI`7NQj!k;A?KDEVTyAdT-!#4;nW*oCI}gPSo+PKFyC84**Irn|eTy zfrJ6Jkw`kT2lo@nADm8@XI(K6-q|8#Z$+;?e1Y~#>Kfk-(x3}^5g4rMmbFbs*BWSX z90c+}4jHHVnpD^6l=oyJV|JacU_pcl+)pjo@lE=4wFQ=Jt%eQ*b5kAN3nM zlT2*ep4isJwr$(CZQJ(5wr$(mC%^xB&&7Lp`m$?Rb#+zmex6=yeLw5+SVb|TX5FQ0 zUsUxKS!E^)RSk`|s*MFlsM+)LAQHhg#>cIkPuw@-=G=j;K?mtEF0LYO+7s@sU{%TB zsN{^z*}?1aQ`HQ?@N#X?{usmmF^_Jd2|Y#jO~MPyRfb~~{mae$ zAL<|#3U?bMt7WG~kS^jre7u%mlgR2OQnr%QI3Apm(*zwz$G*lDUmcNUOwRA#qe?ti zm}_)F-4P|RH1BmR<*OuLyOlz_S)F##VXtH zVGS@XYlHsR(LJadS`37tIgKfQ_)ypAS9098X3M^&xRB-GMwUS+Ao8v6^?0&UtiwkW z7fcaa7LS{*e3{iWi}hePbU=S2PqZVzEAFq=Rb}iADe?wc2vwhHl-9=#?zNT%r>cdm z&OATo^3zOq=LmY7I0WPQvQ%+B0vQPGoqaTLDRLjCXlr~Bq$HN~H$%g`(=B zVE64Xdwd|+Dk_@Qo7y`ci?r~%v0@PIwX_l+hs|$rN9D_-7UFqK& zUqvV4Gnk2L@;J3M;+o1;I`jH8*C5cMK9|`@{_}L~^g2fL1f^N91=u?))l6EoDP*={%8p+^*fuA9Ydsx3riPOgb0i9Xz`F|9hIY?j#tKq{b#yj z*#|8!g4b27sNHk~BWYL~VQU+tk=m=C3bE_fF7rbWpPL_jkHiKq9&i3JPD!IWlLKlH zXG5m-?+ca&eum>k)rg3mZ)XfC~5InA_!D zW6jDe_syA-|Qx$^>TrZedbsYO@^q80`3;?=g`MNwJ! zmJdW9kgX)6c^0}>mSSga;9coF*YjpxSN1Y3xBzb=!Rr1#WeT}|SfBN`)}HZ z)13n~rv1LojaEIl-Pv}0 zk}r|&T;}Ye#^7N)Ykh8@m#59sAL>#xAQi6W@91{p>~~QWT8jC`9$cie%kH-vub7;8 zc#|MyKYD1`m62gKeK@B@!rJz6yU=^7+>E}nMmIaDv(?TlZ&J}OaF*A2YLGsHX=z3D za>Z3#X*we8hZw)y#Dt>GDaG;$*HZ!fZp?OZa<7%8Q-nDiyGY_12FTJm&ZDZ9k z{JG6=fM~AeJ3z07Bb&gs9(k4t(A(Xuy#w%kx%>Vf5?59AUT~z|XGsEns37`)UEx?W z0zn+M`U42D_vIwxV(0Rmar{>#dtbwo@9k-Q?>9crcISVK63w51`+wYKBf8x6Tbn=5 z9zk!1E`CS(m)e}J#WNBL+6RT*N)*Tmb=sytke~7D@(*OKKY}CQ zo$jDLzOh{)2GXF57OsyBn1-wxmEM{#r6G%xgU(tj8KKn%)7!=hQJSy7A}P2B-tPiD z6+{-L^@wrv($dKB{7O_-a|`G3f`dw<`>K%#-c=f*<)M&D@ekGBz+J0kEjQ#PXSVxn zHgs>qj#uddNTcgR#2Muy%t`j=6O!*I*(1@)CZO`sD!32-UL6f!ZMzZ+csSK_eenVO ze3(`sBkzg~U$h`;9_y|@X0bB$aJFT&yWhmb=#EWgk?$zVLa;xCrh5K6pvw|CA<2*I zd@BtQnN|~exg-so7s<=!6LyMEF7%-hr*8BZ$mb6Wou-+6y-NJhv-bVp?n8PB{^Oo` zx8Y%D+eZxGX@?Iuv&rdun~4Fmlk!@ryFl19Y)7bSxHPAQGUeOF#%&l`E`5Qvb*~C* zi3eWE+S%I%&UOZ9-myIgZRlNWZ7mZobqExp37xWCgC|#Ayx-afcPu#^0x zo`N)Yw2!l(pJFUD4W1{g88ZDdryEr{{eO+=&Y9md$shSsg*|GgFi$2|kA7sbk8Sd; zxYu+79;jLj{B};iU2o;(+FKY~ATABFhMhzxPQk=Fw`1p~x2yM%(g~D6Jm8_EKFeqbA$$nfU$k)U z^pDfd*n=1!;ExpN_fSK0w82A(eeJShBc6F@ z#h0tmDFXfnsO|1LfqzCg6*(tdAGUjf70 z%RWH2<8+X1r0)v$@20_H1sNC$Z-Z*^s(^M_wo*vj$$F=6%OOCt>+u`}k@^Ur&-HES zh7WKipULQd)9tty;`#or?9{zxc`m{;QrTj}jsFV(?Wn;88?Pt~+FK|XEwoo>IM`pV zzAgQfYz9}YT`D6^&dgN(b3JbVbTPcVbXCi_iI33g-f{$>j^-V1%mN_9%HpxU0Sd`q zJ^?m4vC0CWp#4j_j{EoQAU=7vl@&${9m5YnqKx*sO6q0E!-5J$Mw3vaaT@7j5`E&C zVNh}C?uYnqV!RxIO&ug@^}7*Sy`0`}2h?Ste19E93yczu9Z`I>FTV6)hh;6Osh*TMhuhxP@i_cGn`<^e_3&6inIn(=`*D^qW*(MRjVp{o zD?4ikh@G@MJd{&RmN=!*R>mX}xnY@TgD z%Nk>eT%M`fe`V?G*l*nI=&4?nyi&!h8(xEF_;~j!2-Ru?g5N?dSH096Pc+=UXq~@b zwiSphLkzrQ)3$LX%7!geUk65pIeokCjz*?7ZX%U);a^4&?ldvoD7onZRD>(75JnIj z4YtdJx)R8;HKO`c131`lcZr!d!}7XCbWbY~9lUKg=3M{rfr+5t5x5jg6g-Rp)|4%H zSog?=P>W(-i$6KM5H34cNZ(qsfZ$1}ckbHz7EH$mAq9LZhB7MRX8WXxQx~feNUlHz zN;{CL_isID_Gr9Tf4I)CW@%Utsj$f$w$PwTBbkjrg3eVFD;D}~8$0WQO(!SyMJMEf zD)t;-8t5SXO4(+w!E~8b_i9z0!V(!^rSY@&1V%_J~ zIsNkHbu9rvm&}ti{+?8&)hFnJ zaZEYYgq?S@+p~YF`~^0CbLh=R7)W>n{qvW)Y&7F$V7Y^OLFCH?zofx|HeG6v{_w$c zg>k;p^Qr*-N(n#nP_TYTzWi7MYq^59ytmqW1^bVkW}k6zw3K%qTrU%Pr0En~FPaXU zl<*Emv`mM3g1)Hl=YmI??wc|3nUvVROEa9k?)Hlv*v=o3R*3kOSnCW^FM6-cIwAfZNxxg9FdEA6>nhEmk$`IQB*f8PhKX&-?4RV<946Zq1@jslo9IXh>|a%rkO1is2JpUC1vYKZ z0S;HpsEWcT)8O^GaQ=s%h#T#ja0GWo?Wj=a^r$*kk2UmDmRMMmnj@WUuK z^9B%`>=@}V4G(EI_W78yW^FW1Am_gU5Ah3({(xc?^~USXdTIsy8qn19fl2>*U^UV0 zRr)wGl~Bf&2e=E3=}L4qNthJfg*Fg{?s81W$Rn=UQZo;fE?L=*D@iW-o9Hx7`U>S|lTIJ+54 z99Ma!s|pqq%$6rg#%ntb=~YYT`ikRx5B&j^8PR+i4N=Efy|_@pkxm4BBmZ>-{B6s) z=zqpVlP35Dvu#fqLo9w-w#_2i9G`5vu-l!9&`>p9TPV_qY$aoxqG;}!lgq_Po07zT zZTmY+CI{;(T00`t({-)6&3-#}(dtH5>RLy{Rn%(d8BUr_-?#q!+k0XSK-qx46@ZNw&##3YDOdpT2%7=3-2c5)|mUyXWB>Rxc3XZrOw zY-Vb$uR44)W?G(i0xl{$wkwPXDnES#TfSaid%jP$xbZjM09E}^Ep59cns!rm+L4TH zqO!i+6Lpge`a0r%tAs1s_Tj&_rPb%YlT70B_ns;Lu5`v00rzK);sAhM@vq-ra^Kca z0lF@jMyNk!59Z6h-uH)NV?6(DKdEnhH=LvSMi}iPx$rvq`@NY~K1>%(rM`Wq?ulTq zC~Z2N{FGn#AV#V`cVe#}e0Qb{w*i{**&CytN_Wn!s>wvk!uBa4rKs|!mpjo51MAt` zWZxziXMcYS8>C!Rr<-4nPN>ICpU^<7PbahJcGp7?(M0mgRREPJj?4tIP?{yxeM*y+ z%&inHA@Esi$k#QXS2K!a(-15D>4vW?IU0F_S5N_HPHD8l5MeHv0Ymzhr(d#NS#j+i zKmI8*o|IOt@i8STH9yr_+xaEW9-F|H^uC2Ar}9-Dg4CU_fK*w^gl6Had(5o!@hb#r zk*?_2s~^TP4ywhYyzzN$vT?MP%~Z?Mri2OBoUrwek6F)JobG0%h3~g1@!L#py&IlP zB8zEeRK4IgHa8xx&8V-K*vnh)&G8zsX%mH7j-5gg*h zWqfO6m=L&dS;mYhP0SdJ>VY(p+_FOY=WsJnR=L=zm82qC;tgz^IAPiPL%5B?q`sm0wj*L4xoj8Ul zp^=7^1unYtJx4BfoP-KZ*x%@5F4a!dT=(&ia1Oh>V3S1DDGLETD&FA>Hm<0*!LpK9#XjVO=u4p&NAqV0&?cyG>hJiYWU-Zx|@Zy+p z9O`PVjP}2Jhx?8r&V`v0j$qJM%lBEoE4#HR8`T-`jq02+yf?;-3b(xH^QIkHBt9>aO!@^6lrm_MA(d z^4Vz3UNJQDbY!wd?DAkp@8~40B^lY(=%S>GkKuDv%66Yagirs2T|jbn*oNBGQ#>C# zNfzT_ZGRM*B3kLf6sCnQPU%5U&<58%6HICXj!H4Nw?KGN8UjS?`8m>FVf;79(03ajL$nGh&8Mn=Qu0>PukaSu0%69@ zorl0qYu~R5x=Q{{j`AoZQunBNrBzbra}Jo0SR5GAI>|2S{k8 z?luE8PS#A9SQ7q1&%9tjZ2FQ__g4blD5;->Vci@r8Wt>mRqef#=^e$qm?=-Q(DmL? z<@ZohWxP;+^0ni#1H_i8e#z$?z3-T_`{p~U1H_RzjotCnj@O5Jm%{csvEn$SN4CSY2_!=^4L%9%tcRMOT>q*=y|EuJ@4)F-H^1|dQsdStrs9&l$l6qx z&!DFo&S(7j9%Fgl#^RBg@D6`x9TPkB!^n zfB^Lphy%Nb;IG^4%8E;_JT-l{o$_hu4I5_$=q$nm4v%7p?|8m0(;C3?4B97t@C^Jf zzoSpz*S2iH!`$7S_jfFp`}3$&`kKHwir~A&y@2&w><{}Izm%`%u8zsDU$8DS5`tbG zaUm<-GDA#o{-goVNR-2rFg$@s{khm zK8Kop-=y4%tZqp^>IAsm^v}FUxyqJ=z0P7WX5n5M5691em&5JJ#_jQZX@&^)AM=_e z?=LdL^{*zopzx+W(i|-$p%XQH zea3-0hw1GwN90}`uZzog#jt&}(lPK%-9|0F6m$;Mab$XjwhL#V@*8#WVOZ=CA25Ay zJW|3N$I~LnO1(?pgz-t3P1olvp8M|EOfv@1CvM3wdy8@$xUT9q{ihia)Bj{I%rNEPGL-oi7Y3%}vDnFwhTXr<|UG;X`>iudR}HeT?mAc6N`i;@iC^BR zjoJ1wVe@Wa)l0qJ`r}OYy(AnjxY&t`1P+SL!EY95?oak|a;h+H>0yUtem$|8?UrCp zon?LCA3mwF{s}BREiW31I!YMLgf_}}S+s@?SYj55Kf9MP<&$!o^2JsTTFEqs=B4Uj|Kzx6Wa}5?p>D6O95TF}9^he&5%9GVO}m;mK4>)Z+z zy=R6oBQblx!)7JfU#6Xb@e_f3hLIwdjY_$wC8C4ZkeT}kJs=i=TZ1k~S&%L7J*B^T zxjhczzk>RPZH>;3ab-izkX#mG;%!6G7}!CtTOU!jJS z7al{f-O15rf6qb+oy2obA6HbLmh(`mi}!>2U;_TgWz)BJxAlnFchhS#9E2j7vM0H8 zOgIJxWuQMa_HBGnz970mmhTob*3mcjKv=6Se&s=)r&xp$njx$MWePw0X&_(gzU&YO z-!GO{yFowKQ8w87*<)0Qk7eo9j|U=B(V3%M>D;)9+0jgBQcr;sV@*U+o8n*>-Sx*- zr4@5}xWr~LV>xhO5*U)s!YT`k`>uNy@By>hD3F~FhqtXJaKQ2_MI$wT9O{cL>Wi77 z0gs zon3Nk;fSJRsB6-!v$TbD~VhgBtb;5+{C@#+c}I_ z;NT~XuVFdh$sNyBn1SkH)c4DlAc zvj4Ah$|{-7ku;V?G`Gd58e-`?{D>3WFsX#v#CZd~1-{H!_>%kD>TeB}4v+D%vo%W& zl)*ELdN08$YDj7ofbrTGv|xd3xDqhj5R$-yYn;!le$>*uJ8h@G)-jL4B`C@Y;U!3f zZlo*IEdTWSZ5PxwjfQp_6EC3%7PGo03J_tCY@X4fAUt%-M7&dHWgWqlYz2vpVLuF~ z^;=xUjVjs^Rr>22| z#c&GB!EvZ+$N2_G%Q4=4h@~*^Z{%h66%^q>?xU`OUIo~VCMqufPtyRot9Vay9a;?Z zy!61F;(R?(1xy;rbTrIZ5Xk^`TA^lgu$SqzXds%2HC88|l#uO>#Seij{CP4O@Me4U zI1*Z=$VQgI@XxsII?EJ%s_0S_D(f!9dr#215K=J>jloEcHf4UO;*gqfPTEsS%#sZQS$s&?ysC*ID*RtIt}T; z-u=h^iLEF|#d3&-#blXy$E+Ny5HwO#C4q+Jxz=#7K8`y(Txq^DGicvCRIq)an32pW zj=iHP?E$KxT`R2oSug_@<=6rvb6L*>(itG`GwmSu!VYSEO69?l&?ZV zgkS>V6oojP1<;%BakcQ@lxcPB3kOPO2mVan_j1 zHz4wsEEdKDMCXYf^iRrF-1INyEwaJ`v>chk^DsnWt`JkH^38kOag$X$ zjoXvHXPVB83q}*~AREMiYv)FW@hJ9=#L3d1O+d)cdWhkT+;PZns%O5dKuUbZn!zH0 zj*Zf?7SGAZ%8u|3%|!`UAH&ZQ%S(&z`;RC3-6DSj%w#jge&caPGxPFynTHNN13#dN zEGy@8IT6AYp}N-<+S}@Ky|Q$(P7dS2wzdCAqwkCY@!*y} z#N!)cI*twx`XpL+?@(ZDF3@tE+hw8VJE(EM?%lc!4)zGv+YiNh8LkQaBdqahf>l) zO&=wx;f5T73(qpBwv?EG=){@j!4#OKM0O6R-<7>KOpab^TSGuh=d!BF99}98kY0^^ z$H-OnQndWa2&s*7x)~2aR0iMm`>2`WGe^EUnaaq$rswMQ63XFDeGav%l*GZaL?(}I zgF*9+O`}P3%w=WBNxuh=RhVifU z4T7f}X(Ct0nYx^=SR`-@om>N)Lu32~FPHYkUZ(|rEd(_%G3&~u6}?N`Wih{40opYF*2>&GRjy=bW^Gpk%B#l3qx8GUEHfVB2_J}Z*O#4@e>{U9 za={%YOb=`<1JX2z(jPXxhgHwUs*QGGY>WbFdhpHF8++qVpkzbVg( zixU^ZtMgj>3JsXq#mh{CJNn)3bSzH=#a?)SOW=A%Q@!C&or82uHef`Bu+&&UTXcP@-GkUQ#@ZS`i*~gh z#JeX&jq>Xrr7!G=9vV|`~0Mn13vSJGhID_4IwW2HV+Adc~F z+pVv8oX>B6=q-3*9(|mH#}Uw)?1K*lAoGo?o@cPqnwZ9Q;HcEzi+&HPmECBSIQ&}j zCP{|a-`|JV5gzDdzE9ZyZxgIeL|kP@eK{80Bzg4N))eti_Ng}}i@)+702dNy@>P0} z`0;7$dhdn%^*34n?6m^M{12B|vMM93&b^p(WB(=$7q>924qywGA?Vb5$Y)}{OYbeL zjY+9Yh2O$@F})R^Ml9wozUSW@9bT~_KKtlwq~5d1MtRPtBo9(wX1yHlTyOQ#lJP4gn^SWb&DRNJZk zHrknAGg@O8U7HEUlY-e)2(@4H5B(Y%na&cAvijwxNN#AS(BK)=hX}jnLd&MCyM;n3 zBHZrZaaEMzVt-jJ~eNJzd< zue;}e7mYesf1s4^#<_jIcXuy8PXJ%B2tN=0+0uKfTqo7h5(Q}qr}A2sImU6pPVUUk zbBaGr4{-P*ve>{w%W3)WpTszn@@Y03od;ZI(oGGzuX&J~n)a4o*lNv_>(b8iV7-Tv zkxQ`?#~q8m4Ej&zpIyjXp;ImP?kXuBYVxO+cdMBe(m(^phLEv_9%z}gt>6uWa%X1; zUl$&DsRMXV7)M{2yj`FE@&y`oJl)i+8Vs3kqc-|t&9Xu~ft-h%G~;m|Z@CR6SUOe` zqbGfA?&8};?tgG+XG+jP9jDG0gJIt=O8uzP@(%b@n63LTP?>t*GJT?h!{UVNzHgV# zr4hExo;qm1qXY^}LdQ)B9|fN^_4mOiS$~W_xeLqC`tRRX97gOIa2amv@v;aDAX6V= z_400>N2oif_#XO6*aXu<XMtVqK*_grbeFr$gG-vJYvT@ z@u?m0mzyf|AQHs)y9be~C$@ZEfOq)itG7)XsMK#fug_XlfZ96T8|kCl`uT#H_Ls}u z1N{lT4v()LLwJH*0$blEIiB#GG!zLxxed1DeYjn}F?T-OBnEtVYMlNIhSSATV2>YO~H4|=S0(gRSmko@{cN$ z({xW@E5np`Tvh4|&ADqEoCrbYagvdl@eAmHlXM!rnzueZ=i){1Dj4&g{42xGQ|iXL zbl7GyXe_-?b+WFJJUJKAP)8w@oBTJ$Bh6^WUA#4vqS!=RFV{5v?k!v&Iv0|!~iLI#GSGpoDKx?!Dj_*wv7FX;6)&7V_%SmUx zP-heuJ)8f5u(7#2?*K%xO=5tWC_u0Qee~g!w1uh;=~Os~~jR^WNq!Zqr}vwRl28e7=D*tN_B#v+Ru zYi18=&C;gt9omB#$t7&D^kzP-qE7u(M+XQ)P0e}P$C9;)i^-Ty!Q~sBLYGlmnaxmK z>3cKt(%I|q+PMzYCt9BCTVy+1qrM5KW3m|mKeuE1T4WD6 zn6tY*JknXFh8%_*Awydi94nVhT8E-oGA-Ru&m2OJ;Ue)+Bh4< zSR3=LSkFE@7+Lzll)$5PIOSVl3`;7kP=KjxyR}XZlV;*-^j|^Km08qk0#~*6Ysw(t zLTr7oBp!Eq3*W$7dxX`6r%>!xqtO1N?f_p9 z|5qZH5t8cW47P2c%F@y`=I#@l!e*g{i%iUoZqOkW2jvW~Z$8hHnd8sHY@M4k;W#HI z6LZ=(8gcli^Ea^%`brZgIJB~#D=vpYwA>f_K6s^agcdQcuQlU8usmARGAd}Pge;5C zM>q#{&DS>Ho_9y;H-uFZ+B9;@ZchO`jVeFt!`{8!B1!;!$d4P#w>8#Cj!S+|nXpmDg2{ z35^%4Q6l5Q<%!CYmu73c-Me_v>176&`=<*Yb7m2)U&DH}cpxd+=ZKYiBww$;%nKj=>lQB6H5N3V!E2BrB(TSw(5e zOga3#F_KyRPHm+kiH2=r!LMG<&a`!!J@xa$vKD~$vGf0Vx);&)763k<#VWwdshgWq zFW|qA-~T+_nB*%v<+(Bjxxo(F_r5hHKSN$UPF87!Eez{*(7NMiD?53 z;T5=mqUaYE=xm;ZI?9wqo>78uBv?kacRml^ehx_*<>g!kaobMlwz=0THp|@2| zDZuH|60eBx!uM&ad$@7vo6pzz{eDFCn_J5@>gWS-YV>+qRuA|ot=`E6{A2^0BRsjL z%PXKOrvHrBXY_h}$)XDR_yVb-i2ff(ckz$1kQ(c`{*~qQk6BZtlZtgNe=$FfAt{(R zOQS%$-V{M!3Z37KwCC3K5%YjGBqzJgO5TE^Ch_)PCyIN@dol zW3yt|@FRO?8;ug-06V>MO!%S18zh{jhIAASRRu%lW+k|yHBWQGFXnut+gz6u<#mSu zg1Xqrf0MHump6E-42?B|&r8D*1(6tg1#yK(K`_@-DRgbfto9sy60s-?A}jGG z`t6Bn*aQ@kP?b@E(e-2(*=KF*g*&A{xlD+~h<(SWkmxUakpdwplXimVPvucnBHj;6 z(}AF1|D6aBBUH@$H{P*iTXOomV;8x)U7n&(^j@t+t7B&7UyM~`4sT1-gGmtsjY?{8t6$E;R}y@+UvH`(dKoL|%_Vyp|9Vb;_VC)xFLM$oSY zbGqZGsXlYW3Msd^rna%43d)0jWj=Kgz6wVG|Lo~tGSK%=D9pdS#J97H3FWEaITBZ3 zsq)U#jR#nEzX)VNQc$~U_q*TKu{_QZ{!;1iC}j*sk&>7`JZ1!o6M>LI;FOxydXI1% z1;MaZL`4eS-{vbNj2kO@z9L4%fvl5r957pNM__&8G3L7-5>j$Z%n1(rOsL5bEE&<{ z7QNxP$3e0tlYq(SfYH1*C@Ob@J>=!dF|26AB(ow(l88AOade_3=y%P+M|K&d_B{Ml zCIOs#h!OC=lhCXlZ}T5;-L)0*jIY!Q@#&rZr0GEXS%W|NpAj!dmV-JbNKv3%6>Dr< z5F~xt%wNj>L%$Bg?1wgZDKvg~$|hSxZV;D>?iSHF)+le0FQ1P9#;M=VV<)z4L%}LK zKTp80#Bt2enT5)m8#PC4{}dV zF6QqZ88ZkS%8uhyoC@N*8G+aa5Q1%l{CnpI$r5+y;o8L_3Y_O8cs&ZWlwJi zxGjLIY1#rJ;3B~sz$tmQa4b7D#)F4Q@HXW_012!@SiRG3WXuzK4`xCVP9)sz-d5n znlsfe$mL+Q{;;o|=Wi%6^(Vm&pqk@L9ZA9T90jHcsm)MUQbawiD_K_}3%8~_?7W;& z+4cVn@=BrjH`3@VVr!wWK%3(^aFgkb#`~MxDOLWDeVrm4gsKod)DA^Drbeh%!abLP zRFf471a_p_oTg@t1H=GRcN5;5)H?7cJiQ2Xi0nS*7AI!6m*quN>r?nZV{(}I5bm~5 zcn{F|qBpZECykN15AUn5w(O=#cb+8Z2bOH5A;~Rz1xv^}{b%P;sZ%HIq7d#+(P(%{ zSu4>MK)xmCG%Q63CXz0tp75@jE_K;C*E%e|qnr@scX~RPXqw7itTN>Yp7ae&!+5`2 z$#zQaaMqEEw8S@2AtWXLr9ijlQPwhIp)JT~@DSPE0qR}E7qejRD+}Iv+AAH1M27|) z_Q4YQ>E5P_K-P3dFSUd!^hU_>zobZSHEzz3(#~K zY~t?btAh0K4J$sfy5_?q_RvaWFAF&@)N)|=%D_b98cB?H?~=D^<7&tc8;G0Ek&;wZ z5V11M{@eVqo*U~vOhv4Yb^x}+=7RuE$NPcTo zQz^FPtz_jZ3pD7^MZ>kx(;u}%?H?DnIEMizAt?tIq!h_Dc_ja!uOKw#^y%g(zWE0b0{0vbbg*e6-ulTj>2RcObIXc+N(K~;=OK#_c z$v^!@{#740+hI2)_~>B`nq544S-e&LPrhQYNQEW^YFu z@i;e-stP=;7th0nC(t0i9>$Anl>k%lecoxlu1w6#DPGr%7g`kB9SyocWMJJ7DK%wW z8+Q9g>7#{0$^VD7EI{#-Nwno?C*VGSHo( zWys46oqJx5FG3~3a}ATDm!*VQA(0B2)M9NEWn|4EdLPw6 zC)o;_gf9lPGvc~3z%xqW!g0Lyp zWo4wyw_P10(Iqns+fcnASBJV~9@}pMmvj-j0xh2nIaf%Xn)-Dzs_2wsK#~=a5Z$y^ z5p?YlryaWp^t!au0vWdWkj!-5njyH ze3fC7n2!H6OUMti&e}0dXC**$JjGnog9D%0&O5+ch2N-;yYMqkIOCo`^fv;=Lu1NP z<@P_U%!+8`#D2Jr(!%k`^0QM1!?<`kjT?i*01eQDq>LiMHi0{+tDJwlVw*qm<3)KB zP5ine#g)W*viQCk-dJ5^hPY{*+R(R^c4oU)o$H+~hxQ$@l{Hwa$)^ytaU~z|0gMym zrHOutKk%y({qb=*DTl>1Ivkb2L9D(&qK^vmeWeXo^`$$jZE^&D#bQ0Hrx>4zhhi{H zKXCDLwh8Yz*^|?)lN_jVXR|QSmDHN-_(F4>ve>8tHra`2tR>Bn089B#AyP07XsNkl zo?s`wTG++}NdUD}M^H!Xallk?9##=Fy#q`STATLp7%<*jT?N|AMC`8g&+y3T2M*^M za6>VIi@-^J59ze`y>|vNqqz4V#F{?ODQS%SbEslC<4|`X(+oIAGcnBzNBc0{f8gsm zkom~TOy+75Fnc?#`d}CGNprGk4n`_rXamsk_{>bfECw(@;{obw+lud^A7cYwZyh(7@U(v}2-6T*=ov&nx?h26jPE@r+QK z7t(n4>3^;AYyr_UU~GiTL_1cnH9IhK=L@KsUV`?oTuGszP`U%utAFHiAXpIe!Oi9B zviKgu540FmM7mV^H@U*xt2rhAaet#HgsWxpw@o!m zUe*J)9GM`uHs+~)eXqec0_CYRx#ptK{`0hiUhr6b~|4eUHn_0bmNfmIfm>1<4U(br}iJ1eO29z)st2H3tX; z6?X7na7(f9kZ7p91)#JKniy&l%fyANPvvB1@8!}YrBMN|Oz<^obbgM{Rt-<{ZeCHw z22mk~&xb#Uo7?^6+U@CDW1#?|G)3{_oFA_@P8N51UFkm`*5&QY{1=ngReO!Obc`5k zcdOqMm(`dvO3&iuqAF(<&;rw>j81foYB6CKY>h5|dYqrHbw2E#v$|`G{MkKQX9a++ zJC}s?M!A+m^agY11McD*#kd)Fx}=e^7=#-tW!%I1kCNqM9dxFPscinQq}1=K9Q0#Y zdV1uuNJ&R1Os0>Dg8KiZq`0pC@08TRe<`VoBy?$c6Hyg`6k;YE93I{TzEL6NX5xg` zr`+lH+AY96p#44yB7J4NyZZHD_Nd4RAwT{qzSEgPcArz>Y+Bl9dx}u ztZH@)7vyUg&~M}XwlRHFYADwkGue|(Kp)|z^vu*Qg$EU-n{3>uB?k#(dLgE)r8dLh zHcP2t&5eto3*7%wP1huwDlbjuqt>b&gT}zE46CUK9o6KcAD8SIiL0k8VeZN)uOYTl z5WOW`Wj^IqO)-)Lua|RU%%^uxK}q3oqhTbSs-{UNh@3+oD%V3O!p=`uxeR2yh1B@?$XOkj-GhskR(0ia`&k zP!TKPyuiDGiZOP#1eP#S`CC8P<}h3@5JnpyK?&!1m^BwSF$RQ5?T;5g@>A{6d-|6@ zIx|T^?*C!$u7cuz^ghocf#B{C+}$m>y9I~fPH>mt?v1+>2rj`bxVyVUaChnLB>!{f zJoD5{?NsgV)!y_y6wtq--{03qiexcMy16_S?W@)5b@q&blMGz_Bzkg2rIxB*F zWXj=?Hn*pVaMzy-0oV9=oJZm~AF}uPVqYSSw4~7l>z(KQItedkxnz(r`3iK&GMJ7aPsJCXR00nizU77>Tw-}I z*H&GH?_02RDN_Ne(2**8j(!Dx@x%%vN~f@0?M|+cT<%V;h*>#BdH0;RJ}3vFKVW5r zZFI3vk{EO{i%1l9LvKPp*S%c8i?!z|zk7%0<3QAu{(I_`Er)fkvcT0fOSFef4P3>C5&J-zW>j1&)>mowoVR4QdaYlO6? zk|<%4HJhA7iKhW*@7=Bso-?gF7YFse_(lW*wD(@t8`f0i!(e$bUo!_Tq2@KCX?(-k zA$5=zH?d|H`gXC1-xNIzFVV*gtWwTf^~}#c-jXN89DUbz zqz`RNze1ArVYoT%L*vC4(gk(JAxKG~K$;&f8MU~ax^txxI8Ozxwg?6F(IV+_Ifq7> zmRAz$jA)3a7&yritdb--f*EIO)^A7!KE=}v5{0DuF2pJ^2#qzG@6{;c8cPe4#-V3S6TaFPjYKCl9`9s5PEp~B~2%)CV_QROBef| zRmk7LsJ2@%r&AK8sCi`E)2dW9-3`z&-R<4Cdu$ zx*{sl;I4#tRv*_8N=2W#P_uwU_xek-05uI}mkk@Rm7Nw0zAi=w(`;?GimJ)-;)XkL zuzjBgp&n-QOy7siyZNJUC2$Wdy-A(EVcoZW%&f;P$7BW3O9w$_X__BK>`o0^H9p1p z4tQ4TGEhXXP}$tI4Mog#w2r0E@zz%1WxqeZt?nA0L#bO^P7`g^P{ceix-Qq$Y^gEm zqp~ealSQK6&R2yHVtd(4tg{ zgf1pQeVKxqZ<&YEhYwb7t(p)YpBrQEi}eC;Wi}&yhqa$13aon%lsW6OI^fYqx91)e?iq zj4)DN_RDig7=8_sysP?HWphRLu7c}Ay0MnfnrxIRUMzfZZCBG988$^?Z&psnUYnyhVV-0zJ_+>bsea6(*jtudtQKZRd6Vb9FTE zljORFTT9)#5NwLmV&cu9lv-kRM$JS0v$5m(Dh=K0{pWE=9ZD`!Nzhx<8#X89iN`v_ z;3bnXn#9menTjc1of9W28J)HcD|~3&UG_4LnO=7vqx9EB!z{d?pBh;xVOVOp*r%o( z|F=mD56OQ|VqpJk5_42Ex@S~R3jo$orlOcP5&xHKCx<4tQ06H)5<(0hP4Ee$lH6qHroxSJ4Ckg-l`c`X zYe!<+&m;u@yMj^rcLifHOI9(hv-3{{BcFtkUM3qM*!I=SEIg{ZyP}$kYnnD`#@!MC zv?~Tyq@Y`TK&Oa)-KgP;@12ELzdp*?6ACy7c9*{ z+*e=RBC)!BI@j|f5wl~v(@EZk8jG`KZ`na5}QG3KM9$#g^wFl%)BK6FX~Q;UjQ zAELds2QSr8pPz~*JuMK|GyBWdk+p@XFkD~8hq)y_$a$DuB=pCItQPXjQm)lc=AWN8 zcs6{Nm__0Ew(bG{^L@;QHrKD7sxx%JM2=+E7#wfKW`!-vA-3A6^tIR0?29&DuQwas z!b#X%#*DX>uGxLqfxHi5!T~ap$ym$(CV}xPc<|?G`=$DJGQ~V9-YJ;VdCCB(heM_av&y%Ey>AG+TxgpS*5My{7*M1DLR9sn>qNSA_WQ^PR0mk(fCi z8l!JC^DbJqE<*fM<@2HgZTMB1jyAR~y8(_l_%D#Q%HBD}PHaO^T`wxrv&){bS~Bj! zk@3$K<4u?vLW}R{EFd^ZIVET#eHhL0Yd`fhF7q=*!t@g>*NN`cxdSLcXPC^F5c}yl z(_$sJ^*x$Em<9q?0t9GzzEi2s8=ly;H99}Pyo@|X zYD^Fj$mNqK79uj=hrd^5M~8r|4qpk0_a@b~$5kvrWRnoxrZ8D#>z}%?WtB6IFOy_B zXqocX=03t3!6`RrBVO;ciVO+#p#(nWIDY6T2DF{S8dzjx!^`_b!m<8vUHYlv7F$Zc zQi`-vq6I-w-@Qayi){&=Mpzne9S6TJ>_k3Qb9r@o_^7Yn&SA+pNKaJXL7XE{Z;ms% zZz&ZM8=KN_{_e9zz3BgW2xF%zhoRf3=G;mTlg=X!lEE{Ww)((gC_zLK<5rDCBK_XV~WNLhPnv)k!H2T>Uh%;vM4 z2YwW2`PURY9MD?)<}}9ioLI#UYJ&2Ezh~;cblpvK!G25Vn((Bq+re!r&vrSp1H@LKNXBl8Rsjj6c^0b6=KIR+Tb^3$c3ZzhZXlMR0g{I`B90CfpSyc znT1o*`EvGp4{zb?R*rmlbs8dH>TeZLr|cS@S%sCOxpZ=C>mQ8FcbCh>7l#mhlg*CW z24_eSr`^TYJANvWjAA-}gFbmm)ZsZ;o;l-uw50(`uak9ZlfZz!oDfA&kgou(7<0bY z+3IfV1}w-9sr-sqnDlL&7D@fb!F$Vbm9UFbuE!`D>SpLxEkMjAe05e#6d>!RxO7cg zz(;p22e3Ou8v4feAyA5^l(q;g5`IHHe2~134d317a!zbaT*rP6TB;1T7SiEkCuhE# zB(ih#^pw+fOCg|24rRhxVU@c1n}f+NWKz>9K)&;0&X092GI-i0fntFe?=kTP*D3cZ zyTgxew57rI0B54pf1C5P#n>_EK!o0HNEqlHgk8_%!mDTb~g+us$9ya z4xeS)%|X8GQ=Af%`vQP$9%e3;skL@eQ@ma zL78C2knc}y%Pu~x)iwAl`+7|K;1@1}O!mFbfu7&T_-DII38Aw;h~Klw4hm;}(=e4- zH9ZqfX$sLF!yE`bzVkEzW8EgSh(*qgF{|i+egl(Myp(K|y*gen6~LB%kf2f-s#Kfd z=J#Sy#J2qrMJZ-Rml#NhaVQS%GGKaj%zp*s*evBh4K10<8J;qGmi;F6W-mopy+Im7 zlVO^}5r(u_)dDM50WIyK{mU_pXRqm@JG|yD{YS?Z$t;};BtqeR-^b+}) zORHxR2T`)^-fD-6Rm<}MQZ~b*Z$Fz0^lCZ~SpnMS&DZf9*qm@6djXVxdGh)OF+Vi% zN<>X%yLJvLdtX>y>2J96l;u_%*(fRWKSR*V;f8jZ-QZdGG`5g{K8vr%RAbXt%YQoA zmQ&}OucxJm9J^vSpvPEUs$d(Z0KP3BcaFc^o6~1Y&XA8SsT-Fs17nFeu6>bWLIa;) zK{N1#JO$!ij^$rMBcmjO9dP@Uk~j*wFtb?CKGr%i4Ul?eUf@O=liqPHyX2dAN;zW9 zkhYkiklVBsOaD3n1j*MFo(V%sCu)Y(Oo2Fqno8&NpD7UJ@hsG%1KtC^9+zABQcn9P z0Bj}B;Fr@rDA~ws>0+YVOz{J%}4m~x2ARO+g@iiZ)7uLTf zM$`h9*`0A@yW9aE%VtspJ3uFr$;R{>KuM||Ny*z+LGP=FsnE#jW4Dq=r1|!czcc z-2+{RXpwIhII5W_>bEXH6;2P!NEz;9St6@Fs_lL;MB`zud|W;wblRMAxSvf6v26XI zI2h@K<{)C6P1|a&mZ_^lJl6Yr2EJdiDPhop8NEzp4*#G$OPN;Sigj)D(8Z;qGpbNlpA^wikFiDd_nkNN|v!>p|iZ6F)x@y1S-@hgLKl$L7(V z3sIV_MJWfljyz3npw;zEyIuzRcUkeQcll@X*vAG()ODePD?KZn~`npbsL|?t9Wy zr<55C(t~%7U{hxfN=3~ib`{j-)Zefa;U|gwF}tW!Xj++Mg`Y%Qf%6mf7%_tFDnvF< zskD-D2)xxS{AUS{fkHxj|7E~CmV$5;oEvhlfS7!U5%&Q(nhU|OadJF>Qoo01ADYx? zr7I6XaRHnFq9(Yy%88-Gd43rZZ3Y8&oyWb&WkGmZO0>+>Tc$@G>Njtw*8WW+;dfbs9>JZZ( zx`h@l0%GNZh$sW`uJpUXxTF9IiD2R<+-Rz~GmR^3+AvfY{$<}1p>8%k{2Y$R$@=l! z;M@dC^pA*p-yQH7CDf>1v*S_w<(OlX@I81nM8G()cL_jfgoBOckA((pr*`h;({lI$ z4Gn*ESJ9I?F;wo6ZSnRlap8oxeUbs=?O+023oONY8E=T12xDn6O_Djy%t)p9D9ixU zVe|@`tzi!sAtfKR$nd#-aIgZf=j;br54udnT{%P{0YFOBdyjY6<3m(eM_jxUMiK%% zIbl?Uq`>nzU;uQA_I49_4h+Fc%Y0e9)&K(9?+JL3E9u`8@ZkaoREZb}{!*vC`oUh7xkkjCmN5uzv=S>LaMBqOjishj2j!usb=TFp3Fy$FDXmIjG8 z5k=x?U2~kFAkzKkjb}cFe9bA6&PN$M zsufop;@A*eiR{dz+rB|`jv9PF@{b%6^gL(B`+jj{4c(EoPt+Nx@N*Z_NWGX;2MXZPIKRn9*Trvg?#mom)1(m-7RQwdgiD#1SbbmrTZ>Z{tiAj<{LetoA|;{ z;zp_d)!pwa7;kG(hHpG2-P)l2_q>C7Hfk)~vY(BCi18$aO!8ESk)A;k*pI>c2?mLA z<^W8YdYb?eU06^wg2RO~YIJnPjFzlfAdbZZ0>?z{-*k`6X8u2K!w=PbyODCwK9qh}lFSd9nzy;vikxm4XZW z@|L?HTvZ1rX#PDwFE4HKnzvBt``%LFzGsZGjTYLcBnUbFjPEF`N#KGqE!EHBeu+a8 ztp>Bwp!xSOwT3ALx#PZ@})z#h*KI!8A6YoCdr`rcYLd71{9KIHx`JRx;={yOyg*y_YhjXIV|=~IA5f- zgy$Q;ovUed5DhlfpVZJ3(4IoFIFI}2Gfxr4DFe*xJ?3G0(8Wz2HDC!Ql#=8 zCs4Lu99EB^urDf!&QC4%^gIp&J zbI5G3SetKEwUebzc@po)Pi4p**ker=#pMgxq{@wa8!E?i85dh08Z#>&P#ZV{blBeV zl4|F201i+skgj#iGe5z+OR=4w31p^$(SH$V6 zMkZAvG5#|@c$D(ikm-9*P0_lNC_amBFxw?(BladTXjh9_&fl ze;X|z-~Eb|bpy6E{k}E#T5|*KK;The41iN8U=(;#O?ZPg0NQ{5{(JvjIPD-Ha~U=H zIaAj-Wsnm$)B^lae19xW9W8*HyK+&a6@8x_{U8$uGyh#pYfrlJhkaNKs2SGcXmD1? zD-zis2W6w(veK!#Y8%=%*;zjtL=?Ka{h6(BM?`TNJZ<}`^ zciHi%fXm#uN<1N-tmK*_*L5ukCQ@1=&>h)r>Yeg zUFT+ur_%upjSU?bd#UOvki35e36uOEC_?BEB)y%8er5P>`acgG#@e-ko+;$|0yz7Bj334IqSqmLr8z|9bKOiqiYOvjsu{(}$W8``Hn z!4A1Vx@7pxO#X~7?d*g^#%VbJX9kkGGiAt*453ahtCw{@TN1xYDL}2oN+ixn8X;?h zKACL0WvqNH+AgcTbwMDL-~61;l6ZWb!5PKx+4dF%2lV$373Z?}hk^I*398~K7gNGJ zDg?lZ)h9wDdk@pug3J`B?-BSTG@4zYhkm?=z!*ZccV!2`kKH39Vo&+g1R^1$9iVa`=Y|o_E=MH){z2rtywB3@1bz2 zCFqbs)`D8Q{d~2kZ5=iH;JVk?(r0+?1?^$DW}Mi#BeG@SveC(}w2`Fc_ANqYw6PIt zxW*1sk6W|iJ_MHC;((Wbgl(MTx50BCI1KN(#wJI&?ONOJROB@|Y0p8mt=6yUF>uI2$h-&dmcGVM4<%+$g+5LPe*uv?x{_r^jaN<=k-5#9!kV#~>{?k! z%`Tf_li&?W-U}$GED+FkbSnXJAHFJti<-r@$!U$T6f^FKkTHKbK0YX|16rbErQClj zrjB`k*nvS7$36h4o-kYd(tf=HRW=>>svWT*UtDUC^%IF$PAevX5>fIvo#p#ZaBk@w z;QM&Xz{db%t#xgo__VoC^lX)x4qs4=A`MjFdI)*U*>@dTh5BW3Ly5zwzppYqp)2iB znxbK-gt3vnnvnZ*OfyBQi$6=}>Bh{4?GW(%25@Y`p9ugJ5G)$TlKmFq(k^>P*4ys` zg)|H-$_{5m*3=on@uF2R4Apjj65ADcf~%Iffv}s{h`g03aC%r7r{u`Bc(1PZKz4mq zD2*-V?-2dY^jJwrb1Q~8W}A37bdaBSFG>4R3t^=~)}k)CXt8Ll4h$<7?2`NiJ;vRW zjZDJ*Wb|;hJf#+ytmJT8ihz~dg$UJeV5m|tq}xmcp+T+*W*oZ{F$K56E7<8(E=ggEtEI13;NOJL_U#EVN^W7aO>*hXwloG6;4D{5J*EMtD!3u0= z`R3Pc^X6$0mRWD;f!)@u{0-)aAq~oZz1=MgX+T-j2obMbqj+S{!cBz20c0IaT8@M& zy;j1F#JSn-7-OTL7A}cgfWLV@4qi)H=+fV8TV&37Z$-;?-~^fr2Y{WW4IN`W;a zPwL87Z6?^H&Lt01k>`1QVb`y6GOX%xX})XzL}uW4@$v)!VL+Q zONmUm6sc+DOCN_Du{Xj>${4dkNqL4$j~)E!Ampo)*HD%yQ_|5m?VY!uKP2Y#MNI<6 z<-Rzu7&0`jJ~icwnV~+83%SBN>-YHPTr?|BoP}Kj{Oa;j>iDF;3QQSEdnfX{M}EQM zC*LxHNg^})an12pQk@yhwr-%8x8v+6@adO3DD}Jry*vEBUBaZ`ix2P9n486p_jG{G zh}mnfNF!Sf7C73348f=&(*wyKE8M#%(XlYdw%-y7Ua%AtDDUH05y^jllHU(Fhxqzoj*p+((4uE6BA`HL z+mT^2YC8Anf33P`sD;rZ6x-d0%HP}1iLTVmtVI@8zEqV@s2ycIsWkc_ zz!Db|GVKJ-4y~N_Z`-Axy(KKS?!~*{7xgf<2*yRY4nS0#=M*nng|z|k)?mSyKMrr@ zSm|H1OIfACQO62c!+;Bod%)-IVZQn^ZT&ngMVnJdu+i5G{}97*Zs0{1hO~8(^*6=28EcdupeIo|O-0C)K%iuLCE}wWLV3vJ#%it(; zG12GXL!Li23`{B`F>9@73$?AvrXBCw*Q%*@Htvd0TvcaFA<*ooM_S-h=UV~8VLHq= z`?2rUa*9O!gBLxd-}YQsmCHD4_LtYk(oHu z9`B;~C`*$C?yebG<5$;~`5w19lTw8_@A;D@PC=&yyPSy$IP-!EuQGr7F6ob1^8ARa zrH=;IjXI3Y92WC8)~+(;9G@kk$3kpt203+;T@~sPL)f}g*$+u&&w2~I`mh@Ps{u;U zO|;xr8d;@p>esovyt6NEHSb*S?1;`4+o6w>eqsX?>u*-_Z#1ZDTbqGZsBzu=gRMVK zjDAHtZ^Y*C***akVxT4O-#{N|`4QIJ7=@qr>KmOV6}_hDu7iz8yQFw(IcOcw%5V@csZYjXIq#wbEMzTkKB5N9<3AMYXvNl}?xRcRi=Egy(szUg$rkJ}PDn`W#7@#b zS!XBX{$9w?h+Mr{Gfxh@bMAN`K+9BcO> zg&MZNCIf*~zfRLV=y27!%f%P6H#;G|jsar65M4tfyI5`OwG3!X1u zMvAe&gO68WS$xX1Z7noX`LJTxKDOYGnnyS2jWR!i?qWXP&#hy$_d;tyUI>^9qR(mQ zBr#-f8l*f3XcoFv*>|kbP8tMiVvMpuU!H0aewJj(NZ(44U@u3i$2ix^X)od8^?I%K z1f&en0E6pvp=py@@Byz*mv`)1CtP*JKYTj(9Eus)+0`{-nzwiQvk-=kw7Jj)-hd}x zZJbuP<1Y?&s(vb`MJKV*425Tjaikr1$Cy*OrA3V>xZ2TiU<#9yJIDic18NIC92|9Q zbhhbVVYGmwkPEjj;+gNyFPAQ9cFPt_S=`{aa2@*;Fm|9FNOCxmE2VsdE+C06dNJJI z8-QX0|pj&)?%X!Pyp8+Bv zBM}|>Zj2lp|MCTmX~b3Ncyc75MuK0&v+M0-U@<)wcrFc&yEp+8ya7OIuq&A{IYU_Grd(fHl@6V=ayuY8($FC36)D zxI!%sTLZ8(=~zZG0<1%ntAgggG(l@-7|;k9b}NU>YoFgMk4Fkh_MN;xA;nk6Om$na zu0kA+HzIAT=F#b8IWib#3;b*)i07EFue5@&AQb7-xq`EFhoda2s^fV~2S&v0W4vL2 z(CMux*5QrYz)AP(M^K};aA7C4vBJ@->8H9z_OM4Y_DUX-x1AT$7+`cOnQFeKE+Lw?{o-8FVO9h6hkq*ARz8jyR zYJdF>Ybt{@?Tm zyO)^>;xw!J4GNvg!~M^!UK-`cwUuta!x~nRkrr$Hn%!Z(qd}-W9%h7ltqhx7ZTE zq56rqped>ilF{x1rub6Dhh_)wAh@g=RFjH{n=u5ABx;@T3t-3*hYIu+6ec!&pWen+ zW)#n3eED9dzre)45x!m#75i!}fqa=L_f;H_FZ1SlITL!FFryKBN2!b^GYy^U)f-5i z4(tDJvDZdK+7<>|w;$cACu%Cr9TwL)8YCpH$bHh;ebO4Ox2?~Hj#=JkrKH1Q3cJ4_ z{9o7VfYKjl1j0_gs!b7jHh1UGZfDuBFl{G;U!M z$?y35#D1Z5cBLw?rQPFWhb81r^qP1kQB|L3T|v=;_Nd)-53P~fM3&k~3I2pNCOXuD z4aOQ{WW8REcRR?IVTl9TGR(PS{&e9-F@J5DBYJD6UkWj9;rv;aqi`nCwCO^&MdGuZ zd#=VHTZU4?j^~Fwa8VOOqBB`sXvJSr*}s;pb9?irmBosB<=ush9);cl{w#scvDu=! zW0gB5*L&UOcVnkx2BlRyZ!2IV!xc0$z_`+u>hIPUTi3`J&4EB{+@0lL8}*l+iaGpV z>i0kaBEGK|!1aNH-?|LaA6=%()5Vi$t20{iU`CE)4Yd=Oyf?G=^u*3VW$=X%UGwCscP&t5$y$9WPs68{(FE zEWr$G@^v7)IgQxE-I2EhMdGU8TDkPV16_4iaulHA zrMP@e+UL>q)mB~|W>j(a64w=xZu;X~`4pw14J8p-9-H3hw09kdrLAK#^s|?n(6dS(in367Ac=n3s z@S~aH9zVZ7)$g$8n4qcy6xNJ^!kQ3`-(ig&;O(Pg!rh!R;z^q2^NqG%I18<#7`((h zdFN6z&Si%Cw>es=5{R~pK~(&pqXASV%_5au=*vg^NL)%Vc^*+UX-&Yc>Q1qB+(>`- zK+i#$Z$k>uk83)>qxNSj_U@Jy-6r0n9mxG@1AcwGApy>`d-Gn!r(>BgnZYha&fGWx zP~DQeCMhbnG@SZww_4U14+EnX!A~zyXQ#?%7k&~EAJV-2YLND6FC9I|huy#tqg_g} z!W%HR+db`$PM!)H{1WUq-A>w0oqN_K=d(rvR1GYf7Sqk|cc+!iIAuyBeVh^S=oOIs zjmxmmp`6xq9#U5SfFs|t$r~}v&g|^}Ph7?W26R=Oct-`oWqAK~HCCXmror}f`FB?n zI>GOK_7ljK{r%&n!b_*`tlbr`;5~4n-P;H8tutD`)0ozv;$;=+@3^i0`qmyYKr}Pa z9R{+=dcO)9t6a@d{OnG2Nc|#FRn6&bjN|BiC!wRvQJ_XqI>9(@(r5g+Or>^_UJjAKhSdR`|u+h=wv@B=*ZqwyP;Ik!aP{qP5uSsVEWm&s}UgUkG)5hBG6?DigY zds)EFVT@omR|w4%TEe(0PJl;~;BkAgRK3~kgz4Cc1!muGb{<78{h)d)c-z&Gt|G9I zgY3E_CSLK_zCIrH*p59cy6$D#-`=n!l+U69?l}+USP&M$E3CH(?8W5l9qMZ-asOTk z>Dl*q(irI67RIYx*AQIA_vYjW9uyB$`SRT!Y%%O>etkoZfmTGeI_L!69QM5WGUX@~ zfU+8bGEi3I^CzpR5-c?u6g=-j4>qjgijG(^LO9AOB`L>qsk!4YhoXus*Z!Yz8TLQ8 zOcDr}X$Rpl&=e{P0`u4nXn%1TyFa*$CkU505_ReNgUe8Usa^)*G8}(#8UB5807X|H z=@b*9i*%$GB(eYxB@u&FmJ4nP|2^y4>L+obRt_L656{1FndJY1%W(a{W#~-*2QK4{ zl1`fkOPeG8vpOL;W0N)nEl*WGigJ zRx(T*)W(vK0oK<)Bi>aIM>Dx)I8Jz7?DME#+Z#bla9BeoY{mvU@-@sOtv$1H_hUEw z;wM3U3W0IQBfEu@B!qyJ{rt7_92rkZm$4t=bqX(;qmK5)xF!KgBi4<_uUo`Htv1&W zJ~?VOSxw5{tR_~q_SaPUUIcmqBiv3GW=H0p3DZ>? z(@0B#GTY-w$uajiVmIb{<*TgBA+`_6^s!doxKIbwpZ7710(mMxVuP*5k!&NK?)4cVRAk@a_(Z7M8Ls_C# zmv8c8p3jX;Pk!=sSGhdOpJq3sC*jEeX)0o9sMIW*+yY)ZAIDj!R7;d2n--}Qv!y63 zYpg64vxQJqgr3RGy}ym=C`WIQQQm~FQo)Zq{)nrYz=ek^Opn@2V9y~#zRLX;ACx;D z=?4|7tu}|6`W`AcpxKsObzX`ZKrjW^JCTtJxm4kzzk?XUh0Dbviu0&Qo2N4<$3Zli-q4MhP#Q^VnkTr9vd&EQInv1QcTjEX?CF) zN!a^yzu3=tel1424Wge3?q6s9y!$_FnT_AJ3=XWR3L*!o*GU8gM1fHNk31H8ObU4z zu6)qXT_%Bip**6|tHWFaF$GqWS5i!1TpwhR`J>Qpex#qhiJ zcb{My5L0fm-*E$WxtFUL40AxXOzVHzGDfJnR;}Pd4T`KmJW}XLbA#q`PWTS=m$$z- z?74ywa2jik(wJpd8@1A5t{~^DBa$OAD#o9~Fg?JwHK${O+n7bkYkNyj!~_d23;AAU z-bA2SAo3)ikujy*2dsDM%urkAaetsVtC$-NwHn990p3hfg^PRzLztO> z@A#B9UY82MKL~N(7TpDU&2{E>Ny!qBIf`yB&)4fEjq;ksZGsi)V8!Fp2>6pz^)qL# zT4B&9WF;CA51-(84b!1jZAvd-WL>Jj2rHYB%c}U&IfZ}adYMl>Y*mRzMft$@_02Iz z)@t@&O$}4WmEU?C;5df7)Hhx&R6YNxezkT?$tHFVuZw&8BC_tcEW-toW$b>-GVKb1 zP<-@gDEAgvkJr_oqUk`g4Ba1DX63IeW1lShjgs-3iW|wmZ&{{=0){*4k1X@$w=Bc= zZ&}7O?Vfp*vu6TjUs4R19vHpoCOuq=9%S zyEKZJ&$x?sNX>4$f>XT5nY4cW_keVglg(z}q5b(_yemyi>FU5%4vtNML_g zm5ef2BU7L@_Nvm@gE6Q8ia8*Yf&?zB($9e1l{-Dea(GrqwJ!1YCZ|xyqd0ykEMveGqGhd%LcMp%0xM zS3X}zIh;w1E)y&DjT9uyq$U25Wyb%TEaM82Ww=akt`4V0o_I|d#cHrVpz8n}RN)y; z3emrc{$iG8ql@eq=otz0ZBxp3i-Clh&PIfAvfp<%BBb5QrG1Y^J!w%e0Id7cT8rG zAa|A7dImGQdvZij zQdIGo>m=m9GmRL0PlwQui88&=kfh7OnU_fqgcMkbx=NBJET_iU-$fVb-9VqQgQn<=Dwzq+Wkz8SFo@OaPij z^6(nWx@-gaA6dpySl`yID7SI{u8l zBN|a5EW)^@$-PY9Ey18aV1&bjab_~_Ny{^?P)tAKBe?uKQNI$6A&pIis3@HogI$AD z(gK+VOBRcN%Q8j($TFt?l4UZ76Wy#dCy@qZKe^x5`@`X2N5PE+WiOBh%P>q(a%eb3 zS&hVNmSPx`F3AONRw6toPlv4OL;aRz5dV>7%It|ogjDmy^8~3+guT?;P{yv-n(d`zU+hcLH92tQI=}Liwxw#v zFmgfT!WP{!N-&mi5KDiT#tXp5sG7Xv;M)(&@CK}U&>+ed-Vw-n#k*3(bEiwPp5&8G zGkOpZHLOZeCe5*QUvIw;Mz|JC_RL#TO8 z57g9De4*U4Tp4sEHfiQaaI)z|Y;a^sPc$-V6#L!O2q^z)YIZc$bmIMOYASv=HC?12T84v+p2*2DSd?_dgdiWP z5Ib!_Lt6%Bd%MX*_EaHtMsC%~NIeX#ck7N$!bOux34?w_mlJw$urHK;D!WP?E7^f{ zcnpcZ%Om$QG~QDtYVuFhp-oJmCNO^?gbyVBTxNd=V)Z?^wdn?ZnKV8>7$xiFp5NYS2%P|28#&znhwzhu=*N9hu(9 zpQa``(8Yo7Uro*Q@1~|Rbn`xR;^+Z5LkiHNFEqMX@!Q?$EAS&#-Ir@Lr!ilNz42@W;e+l8Zib# z(E8ZbM2Fx-jz{f?KYDT=m*`Dyw@zsF)qLLervCG?upj;T`+=qr@IJVn*blJ0+3Llb zW?#ZmP__9#w6lB#{v?`Rx_Y{{TWx79teHFTwdZzMp8zfsArKyQZj2Nc6!hBXY(-S% zK)zlf^V$3Iyj@msNni6Y1m#i$;Z1FFz#EaseV*bI8TdMb28|m3MALlEy>}C5pnM1~ z@(Zsx_%c&WoYYD@)G;7Av|H6^_+end(tgMhJaeXIXr6B8k|jBu1utd^^i3-;4=hMw zdjFnDpYSHoK!m19@%g6D(Ei4|TMxl=PdBR(^89pj(gE+k+cL@jVap(eAJ5mhEj%P$eK^(>P!Vo|hpc+@hmM>FD zTGPYAP?;aJm#|EZetF60vVOSYHN_mr^!!W9-7=%QOZ*tT~s$&BLC6btmreS*kewu*FH>)q^Y5CQ3W{;@M8doqSsHjyz3^ zzmIcg9CrblqydJqh}n6Nr+_3S5}Z{Xg8!YgnDn8~)MpSaLm!_TCz6SJNJR!)4)D?X z!MLmlx99jxo8Vy5618G>A-QRSq*;x4-}7V;r-GU$NB67zivh|nJcKnXo+PEldL6%S zWSHWbSUk*wDhFqp%KB`F+BesP)0svB%Iwn;!Hs#bB8P{M z#+VV<>=FBu{28x2QLka9-;wuN_CMfbMq+SWGiw|-4*$&IGv||KsTC8VAd0f*r^v+S zC`MSD_Dyk6f!6E{gdvwiy{!5Z)Od32f_@l4L5=S^|L>rtWAEQVO~OAxO%*{ZdGV7P z$d*|yllW`PsEQz$f^3%A=_4QR`xFR=VqTc(gh_iPu7 z7ZwG~ehLn+vWC)ySPG_=Md&C)ogi3xl%JUd@F|dFR`8A}O-et}G}cI#J0daJ(L*f| zgkz$T6ZSG~DQIsu0=P$!x7R$qG4>uIo_XyH3=OK<&K1PWx; zVJx>rffbhP1~qWDguErHm(j;>eF953<}C!L@;A|CL3-uSO*rBaw;A_C>AsgD7h;li zAj#l5el>@Ckh02SO!7X9NMp+8F8z3D z*f1i8^+IZ;@;glpvY-LJ=bwH4*m&LvW)i3Qsput^s)hYhKoZOHXl!~(z`&movPzj3+&-r)ZCXH+n zd$uu8)fH0Lrkl@v6bGsW&^86rZN{#WJ+kh;La$8E__@i(P#NkhcaC;Cmp$kdxueZ> z04Vq%`lMLpzOnh^lW2`x4eoqZ6fS$X*7<8`I zymZC7=1H_=A|+$81BrDsZfTtOK|}MkLyfG`nH~t^>wRAw1nGp8DKsDOSb|tn+=Yoh zY$+!09^66=mu$0DI~)#vZDjA=>P?g7>-;3{bHU{y?uk#omp3|GH?)qGL77zSjBTZ4 zJ{lG)rFA)pieZ@p!4+zgVv0s?h4;0^{T=R!#e6dEDrpTfaDYPGr`MI$6abcFxuDPZ zvHK^|CzbQRHZ`{ZEGgs1I3@r~Y7)(CX$nFIq-FL*L4T`et--UuZE6s$RmOX5{VK$4 z*0i?ERIBE^z0|{<*aJlXOj01pI^=_9nVY+O-*iwv8@RY#=y;?riN!+Dd!1>7P66gp3V1y?*j6uJ-Rj$xeNGq)cieKl|OZyNIlI!>npy zl4|o^XV~1!kPsifa39VbE{{35n(;)kk>f=rFY(l1K4`Lrp15*whTHhP`cSI6;;{{p@f}Wx;o;|7>c|-!?UF zu5F9suB(P)f(!2Kc*{-vm-FlV+1comnp(G?P&NGGKewe>;w9-vvSZc|b1u$96iYBduiQmT2oq`z9<>EM~*w0r%sBBm63#vGm zT%Q`k9v`psufHDfeXjuPx+|_ITt_SH7K&Z9dGcwLM>`Q5D!fJ%Ug~(|_w-i3eZ{|t zhCqDje!P1x%fJB2GJS7lnd&cZWtj`Q>oP!D#v~y?_&;TtX(F&}9y{yzvdna$&ike& zh*W|26Nel5gK%Hrre(5OWkDstaeQWy!^Hj&7aa{K%WP+tZxU?1lS*}BsH|;<_bG@( zDL32O;AV+g8xK9V{95XKM|D85eQX$6S}D@{yOL=ol*x$uc7xnHiDnCI7`HQ-b$!pR z6K?wBJ4MIlBRm|Z21b2Z`>S8^Ug@OzC{w*v%4Cc_Pm&v&idxaAZX(U{Drxn9!#t~z zR{VgVUI=kEZxabE6mbmi;>?Q2-t>DeT^V%$6K(^Yyz8oBZAsF0&GM#IZ^$&du`1Nj zc4!%knN>z*jiWa@D~jT{xsWQu&-FM)OKszHkKUCWODd;UjYz4TZU&e=Z2!9(+UTmh z6xoS0X%ZaOV9l`8wTw(<+{{il9q(9%IHQ-ZZ=)J|Lw3^Wsln#q(F+GYSfTA(9$nW% zL17!$-3%-4r8Q#-)uk_~XXB=;cJqU3TTdloQDBFT!}_cRwFv?;9ZxMA;2V2P-mpw< zhmz6adkV~?Ue5_;p9By`XErzIgx}IKJf`CPZ)q8_^9w*)h6<3DNqhvPWp?!m+#(n< zJ~w4MGVorcj8@D|Ppo%LBH7%)GT(y(bL>{ZU5zyz6Yk6{8}vVYw4=n%)c1% z`t@}8(&b7Z%|v@-k*+gsG-MFyizH6~DsR)2X%_Bhnqb%JtN%Qp)q4V#?42gwKRU+RD={2Kh zPzOx=F=lFNd@k)cv0UJAVHhu9Yxo+ZV=4R1So$0bUk>xF{deOMgoz{*vhMc` zx-Bdh#@*Cj`o<6Nt&&b9HbGW7qTTB@5!N0NB}8R(o!98ThZ)q9c_q5Cl26;8X@6j}u)iFDyk}bN6yM_GD z`pfK>ONMOzjPBp6L(TFYm4LKNC)S^|%wUT^X7_0oh{n|$uB6siUi7PU4}dEvkD(?{ zc@E1!Hz$dA%WtwWX#ezOtI$-nDdH~m@7{Q`mC6^}X5Vb3v43o(ae%FqGf~;AWmd#Q z0TCZg!5XI4JmTzNg=F;*5JWzdt$VGGe8m|pxFLSX@p51mBh&cfG>&e``I0`hlLnBM zS>nmbGFQdUHCb&0q-9J90BMzwYy3{Ua zTu7`Y904_J4b8d3lSo(=3cnbR(s*-Z3=@5pI_e9u9gXXcqRuEC>d|XX@GZ~S5}+&L zvT2o#DaB~Rd~hLIZjDp`iwoMBVW249KNawL-TQq2;BN%7ye-%R#C@W@@1y6sE3AW_ zP{ep`BP<%|ecJC1`tL^%0DGo*N@y|*&~JQOf@l)~A3?0Q*mFra0$}O4p&Jc`%q2Pk z;M#_Gan@{#O6eS*Y4}MHv5Pf1E&-fyDJmi3ce_m;u zUvN>6^l%%)>-{DvtFu0Wu+B3DPnd@>^B?&%3&+ft00?bhe#?K5G1T84za8H9;q|hc zwT{u?W^<28Pv3)i2bSv1rc|5kz~93%R&T*tmuox|Ttf~#vC`0EW{&xcNkC!?1P zJC|o>t?2eR4h+z1%l0R!KhH<*H-?4!9kGYmCJ$;>t9Py0zI0-Z5F@H!P4D$2inxe> zH`ZmqHFrsBjOIY3ck&rboH;}G#;o4wu%1w zi4kk1N_BOM)1rDkv_5zgVXlGNXv+SB-;{1Hwd(|<*0iwn_crX|GkXZVda>tIRZ0^^ z!<_bkOI#bXe-)#Phet?etW`aV0R_vGMq243v0ON-#5R$ek6*4(3e7JW+3$;qfV9le zxGl&1n$$~thWL#J4OCV9E5y{%sQ>~Z;)l0?toChB1XrdFPcR|K^o+Gv9b6 z+?sT@11ul#rnxw#xU;$oIqN49$;OV%Caoln$d`Hop zW!_#MRf{EP!2;N_UrVL{u0m11c$?V48M=dx;D@7WvGLvUgA-nX;G_yYY7PDPx zRqwo#?l=vTm}r!F??~DoUI}%oLGztgLgg~_|HCUOhpiCu`xw3PN>p#W63rX0 z$>9+#<$v=^RRCV;v*ZWk0R;2tXh2z}@Sn1b?8TAFds$|P5Kxv;o&%g>li2-JmWe+> z|GO-Ma9@n0ok+fg!ZxJD^~2g{6m2y7W|ZWHYb&KrBTD0|dGxa^$Jas?LMl609s0|i zC$=PsfUU^Uc?z=_)Xb55iOI3N27%JNhG~m$S!UsM-RjF*StjDWEOYyxvdpOI z6^vCsM-UNU#MAKqUY22=LJ_K4c^O z-Ie(X=*pZEhviaRiTO)&o2gMb=|W;84yx@oMbr5GxWMB1l~(1%LNs2E|K?F1Os1A^l6{-pst) zg&yENJakW?ZDVKRNa@{HO8jFh@mXgT`?x&OyxB@#*ZrNnX3zHm0({=j+ob}Cc6`H! zmp=BaU)<&SUj6qW{evH0b@?;J9PEao)?+URlLphdV+|dwi25mi7x5&C|mTGJ zz`(6CV?kg{TI^B|*?f3;D~16?tN0)oMFrFm6%lBa87h$?+F0pt>IfKc8j`M`7;5v# zw)KA#lp4?EvyG!ukn5lg`O|m;vNF1VvNHLm7s(1s4r{mpGzfV+JY4Z++_V}53o>@Z zQI({b(L;jqWED09BdfzpFB$|5Rnv|EkJ(-VC{B7`;_xFdS;-vUp>qW(^l+vv|J} z7x>;tPTt9+H|ODWi_82%ZI#2w*fl3I%4WeplVHOhr?B7`C)g2`C4&~s!~}*VXe=uu z`Ku};QBn~rPt27^GQOM`1GimhtG)xvev$e5eERCpMIx8gAq~s!c`g3xx)8@w^Rm{S_9IUa-LA6to` zj^@8?C9J<}rR4wEO5yrJBbl_0C3&z$8;Q^z&_zhrxb#PS(DuZm4h?!*ef$QZ8?Iho zsUm5p_#K!{z&8hpTjoE5`dVijLpjBe{puLRqyOHiz z+b-%e&r#p`uF^(;4(cPd0@!5T7Y!SGg=x|Pw~_NS&R7=EiLFXiK(c)JkM#3F0WM%R;b&%*#o^v2(?e zX~smtF$sS@5IpjFTjm1@7$I$_%UIb=_Wyq zRJS+2ii&%?5Xmp^n?zEmxrfasO0$)w<_OR0bepPUC&+M zP9l;ed`ab3(sG1Kk!!0~ll0sl1m$dg4g;H*W-gEq5_WjS+iMvxpJzNWAz;6#lMV&a zZf-@5XGA2Ngz(5$@3NBA-?GxU_nWLlGW?}#P7?cn$V#H`vJ#&Pw&0f`KbN@jwE-;~ zfUNXsU|5flTP$yDd}_huXoT2>+u1qvhi7k#;E;4ZIM^eWwAZ)MQZX5rYECcX54$?U z7urzbu>Uk=PTra_VsyDGB=Wi%g60|IO7g!pOhJ1s^(3@_Q9%p`;N8$HkhqDtMzx^M z891bIfj_;=O5YY(=qLbNlyS_^dInpn%^ZU9)hL~l1x=GyrRAC ziPz_Pt$^wmxpX1)y6&E+<#%NvM2{G0H#c#!n3bzx zg~kny7U4sTNdDPxmUkCuur`6nrJ^Y5e#`dd!)6t^+^}M49)vB3c-h zwqSzA+Go`pJ9dj1Q2ezC&6P-XI$r~jm6HFKm9XArCDFfRrM)* zCWd+095;fy^gs=)_8a?;Fv*AX9uu~&ML{bhd1MzO{M3{&(QH_udi;^_Xo>{_ z0n?w3ie_Yp-4}!1q#PJ-MaoBIxFDd*lp@|UHG;@i?{~H_QO^`<_#6f$7hI!Ufe`MP zxDGo9&k6~{v|CuLJ+&13tZBL?u}>|B7TSkguKr;-z?boSALk;?rC)|74w(PUwjZI9 zL-d2Mpkm%NF|-DOf8r1;US^@egne`iP_EM_RBq8Gn@=+~KkcQLnQf%?z74}^XX|I9 zBVjX6QLa|VWR82{)=Nd4|JjrgeQU}X`!wYOnlhYicg@;>rc4x|DZ}*ElwryO zG-ZMSO&Qj=ri>*Tprsy%w=nhZrc7gI(oIL$TT>?S@Ox1HTT_N!IB>&HasN+KX2xfx z+SjqY6F@6zEz69}HO20J9C5TF-~SPUYrU(>N;NoQ8~R2odF8#+O2YtJNg6;amHnGm z`U0Sp2z2jj%Kt+vdEh}DfV|O4lKZvZ|Du(=+5XTMFE zZR3P=e0RUgkL_@*S6n2tU&rHmX`bwXLB3*f7hz+0?$$A48o289ML#zV(~WOI!4p~# z)U70oY;nr1i7z7|)FFKjP!IR4r)u%05h+iXZt4*HhgPEazVwGyf_$fy&fjSzv5us7 zT8R$wXiN2PTB+cTR`NmYJw?o3IDfT^vzX-c;fRcz>O_V5vCMgpf39)+vVVUm`1^Tk zW4TxOCEMfvV;OSl?@1!k9Pr4w(lDIx*)_QDM=>oiG6O%r`fRF{4WozGJBb0$mLsPg zUKe!OcXE(h+0m6cO8lfTYEySAaRql8XToP%2r9mDotd#Opu_CHmiw*kZ=z;JejA)b z4Jn0a_MxPi>}GrUYzSN$CxcSZ3fl$!5)5ocFxagW9_;PDY4iuuSVN3*NCpi<5Bk7^ za&>6o0{Gnu`a+d^JvW0pe~b9^)o?03+h_Y`HEu{X`?Ys-Lj&M{Z~QvTeR%}j7vYwI zyTz=!d*GU8cn?hcq@&j~?k3Y8$k>(&!u^v$DBkT|b!&>0{Rm3~Ngj8gi9 z2#t~0MHo=!poujlsZ5%u>P$&t?nWs?RutjmnF*0hi`m1456H?e|C5yw;O2H`=XAe! zv_HGp6%>FXPf&blkkiA15TBwr9?VgjTyWz#TSGv=FGkJ`-E4^)#;E<5}jaa zoLZHJw4;q8yw4oI8`EOTX9XY-=30Gz_aAU(z5z>_JiiEmZl4c{7Xx(oALWz{2aI(bw}4{c=-lmWV1{^+a}} zV!u}ZeL20frwU{_n0$%pFLNa}dss?YpNnx;!myFa#!ToBCIM9$8m2178$eYi$cp~G zDg#J0E3X)LN1z8Zo3R!?GpFrF@%#D{i#KGPFyZV_LG~ zA>x5hdt?GrcSW%+7yrS&(?qAURpzUea=ZipmdHmQxBd%QO8WyWQTk9-z5`2u39dBO z`9uIM3U&;#GObL*4z8?batxxbqd>xN?(W5H@M@Ph=D zrn2k;z2cs_91dQr$7u+*D_YqH-+?^aQxseh;|*9k!;=gxsb{}2h}&CYMHJMegkMAl zy-R1A(Gf4-s;T3-6P?zsAgmChO{@46yuxiy@(bvvVpn-+2o|t^b)t=6QB+s)0NET- zwUVmCx6kclxP<@HvTiPHi%tQwAr zmo){tH;j(e_k^Tm1~DFDjo_uX&ub0=tG^ za9>B;0+TF>%~)hqTT)T62G`FBx}>j-{foF7lPQ`|_9JtKX+*a|t%L##qEn-HIo#8z z(@;Gci8PqnPR$R>xBIIk zv0&7PTJ&Hf<&r@kYlYbE5sc#S{2iXhzDhcl>9KJ`S9e;XVVb`n%*v5nrX=tJ$cpf!kFjQu(<{n+oPc3-E{pZ;mP`h0tZ}smVRmq}`}I!EPJ3j#3xqrh{|G3|Vb?|bOIV@@ z2uo=?wVtnPA@0JN7+1VFx-+CmaOirUS?(DToU*?AW#TBfEA+wkHV|Qk4CPceki!@q zKu(otu)sKmGk4%xm4n45GC5I0$(1~fqS1B2iDNYL%0TqRw!AWu#p?@-Nu|s((i*pE zdtx)7w~yx^RMf~&s4BEAS3o$p@#oyfT;W4rHM3c^OO*y3dXP+DTGFS2KXe6cV!&~NlBTG5=c z(4Y!_=~dapmiwUF6)pIji|O>&&)hpx3MfTO8QS`mnzbowsN!DbbP;|gbBwf+MH`6d z$w*+iIT3ZIA>>9F#Y-1fEWXM`=b=iuk&t6oqyE^in6-}L1sSO%+;Z+9GN`?P2{vB| zQS?|NHaUqhU(FMuiQYMQx{SI!T<9ZxXADqCb)t%)#L8SZo%yk9?E;9HhR57@0I`%+ za49pgf6@%6k73Z0xh-S!30CMZoEUHOgN?~WNc?&_}17mk&EwcfMz=|+sAGBoVpi)pM^czt_oDWx31!dnve) z(@aU?%2!93^TI2qx-hL6k_^sDEuH>*34ryx?i&ucrG)2(6S zZ()gs=3Q9oG+O1;j+(wYZiqItumV}P)w0ERFkvuCR83kAT6Kw7q=1;6k$0c;9U`Q5 zx`WxyQVwUn8&wy>MQ%vSu13cfy%!1r;dhXBq@*g==vR^&YAPl8?Or9WXykz^_062i z%cvt&UjWC6~>kEzE`8@>FAcVS&SA-b8Hka;;Q%`>}=z z;bYaNSIsh%*0DA?Q}a(Ar^CII(k*T4EOhB{MvRKP0UK7Q(X7I1m$rw-t!46_FOryS zpeKtAa%>4*^AQZU#)1;PQG^e(P4ym1qczH~xAAF;aS)$YD$VIslY&WN_z3Ts3>KB; ztNe5Z@chG71#Q_K(_M?LIz5-@itv;x&H7_)j#%WLe_!5fY*sY7oU+D}aat6Ymll21 zY9_G$-84?-3Nex9J@F|tv8cSHc5Rw2X*ucV?6=aUL1!_ncyb-%(aJB*49gevNQ)?C z$ZFu?zb)n2*4zn{MT0e8>6>~cUw&UcXRrBB{3R^Oz6(p8k+0Plae%f=$NUSU^Tgw6 z|9-H`>-x-0|KsW7{r_FT{AUHj{(o06|6f-yI=_6596>6_U*|qtSv{<8&RhmRyxv~i zb3c5TV5T=dO46M$9xw#zAdOtQD;ku3DbZz55jO}o?V@NS&0wA;mkqZHc#&K^OfJPfA-8ey(JSF{Yf6FX5x9LMNCkXv_2!3_{~NHx zU_FW95W$^t@l8MZ#~)ycy`OD5Z~c^Atcec*EG;ChJ;1yHOW!%L`WeoO_|&>GLzE3r?*tg`9BQKc(qSIV64)wAClJRDZB(y@T7-EcDTXpjFH zAl-O&XOiHiHodpFQOPowg6cmy2)M;xu4UZT!WP2jB_xcxEm!Sc0p*@CO!$lB8u#^z zMWRUax9ez9*mwu&Qsc{$edi0QhMA8NF%%+cJAf^L)WFTmB5T~l+dsXKo9 z-c%NrHlBZgCGgvs3j-6xXYC30ot!W`HgR2aM}ihd2PR zq@D5w*lD7j71`2jmw8dZk1iPQ;n^Esj%W73KqJTPl@bBQ?xbU@Jue?qtQD9zp}3E) zQGS~)N=rjv-Q23{-saZqS5~1p+pp@CBOQ`D#n^HpF~E~77~Vyu6w<-~#F1#I=A#R9 zp!nT4Nt@C6%kLKNei}Z|Q+@N5NC-HFST&OYzS7m^)6NxO($A|#_lzna-pz=+pfme@rng>SBnJ3Z??9=fqUeqB)Y4YGZeI#yb#c7$N|`u(BNMVtfxXsQK|9O!W;8#W{i@1;5bdxv zfNnhS@T^?ZU*8<%*xIc{+5PDGV&savKWtUIc5Hsoh;f;IRTHMZ;ADo*b2xOVIVjTW zf-BlQZdj>Ct7pgYstacv8%zyn?zAD5*G6K^T%8Qxz&-v0uPuIC$574;ryfKjKa4y^ z%L2JkwskcH13f=dHVB)f^yFBw`xw#8k`?ZiRFA9acZGU*AQ&O zi*49Vi7Pa^gA>4N^ny+vEGdUaEwAh!LmaKft^^9o=&cWNJnx=BJx_LCMHgGhA6#vo zPUirbn}uzkY<_&+JD<~SC!W`8w_pPd=IK4PPVUa=v-Kg88yEh8L!SF}pR?*+^w%XA z_tksdr$eG@_z+j*b4$T~JHm|xkcFyz@FnYgJE|`iM@B~5(~_6t{{TzT@4%A9JFqlv z?+O5xpaxzE>&{nqyRXrKmqvZ2vX_ot@%cUeH}1g1y^=!4d1-<7x?HSpxII4)Ulu>_ zBVMd~d7j+7yqOQSW0^bmWbbgr6t1T$iV2}DYe%Xu@}%23D~_+lvCZ9;PMJz ztDSDa*An_yd_F$UkymGi?5K`{BJkyK4WXMY)=q8X0i9g}=TNJ=1#Jhf7Tad&2SSq1 zA3Z$5Tv%zY+0Gy(+`|OgY~u8)sV`g2`#2zCr9Ky?4V^FJy0zQqo3IiwYvpD*S4%L! zl$#2yENqmqA}5zHW&I5-HOLXvP9YB`OgMgn5%D;-YlEh(1d(*p0su=C`_;7~ z#AtRhMf_a;8NWmb1RYf|3ZEs>3j;qHu}eCwR%x0{1XpUcH&rL30Y)&0**xd35oU)7 zEugQbAFgqqUNv5OU!54T_55G$wwF#*VAN>@ZhdkxT2#7ysyEMvuj(1M0>qdc)~8Yu zJYAq`7)|iV(`RTZ;McK&@UlB+RFx;pRYDKz!&x-W)tAZ=XxYc-d~4wJ)t(-$ zPG9%-*RyNC5Xi=}nrTheo;-cvuiSsd!vjPE{#}CU{6DD#Bk-RR%%W2T|0|@pe=%g^ z<+_jeo&77|g5(_K$#2f#GV+qPb;@DOm&2pzGOUX!k24X2??#3Gva_ft6sYC|mQ$TF9Jk8I)3y^nLZ2oj2cRR}%8s*492{-FvZDG!5lLcy*pwyC@0B^0 z(y+@oGF)y0X*%Wn5Kts#?l)$J!gve-mRkM*OL?cHk=s`|(gpP*HCp`6ycfyNg8xsz z5;*`^a#lE_G=_2hl2SkXRI1k%k6TL&CGLJK5wjezMnBQLqI5tg`0yc)eaQs zHL1ucMuz0?zwf<%`sfdVmK@F`?9&irGEo8Zs(x(TyRiKZENQ<1OMYY|*F_c$z?Q!Q z?+k2a!L$Swu{fKA|CJ)F|8;tQPR!I_R-5X3>w+0Hr`@Yx* zGMI5KhkNdg*PR&8%e*ija!-2r+YbsRcv6^M$OMy(=cgUW4%8+VT+VQs*MWJ~9#`&! z(RetACXjg!A2^Q$*AaVLZoOJ=e877N-_}NBWf+)Jn7wn$_T-fo&Qr#5B+tf7_^84% z9$?YqA(#e6OY=o;(Bry}^=ke@mU2g132P*_w8Z4z0Xl4~2$&QcXVlc%U6j)|XmrbY zV)~oWcnU>H%?iyQ@f7j_Xw&?o*x44Bs><89A<$mLx1)aVzLL|BA4n;=(LL$J{)j(s zNhv8&$tAIVZ@#LE3)r5ZV6J`@E_5OriWI51voTjYN}NH*JbbukuUOY*_v^F3r0dRT zysJfTkX#lZ3d4c^7q6s<2;h~@Vsw=L=9P3}X#pvL_dS$i-ISeN2Mxm_#pX9vJ&S;TwcYI#iRtMrgEcocp`s8YK<2~P;oYZ>tO!M z(TnD1({IvA0azDsMcr6Qf2XK`da7bzEOa0nr1(@+LD-=Vmi)sODCT!OoB_gYfH2zy z_S}jXp0p>QJN$|)5$I55g*tfz%B@Wjx!)6Dd7igRELwiaZzxLO%bC<*M8xn7DU_Cp z8f<3DoR>*jLfy6g4DDU?#H0BhS6x{cQ);29Zk&Gn(1(MdBSAWSQ+Y(h4pA)FqFg7!=weU zB68`XQaGCCXml({okI^$N0A{L1S=akFbGL9t3oAF*)_BU&@2%F3Er#NkgMWC0!4rv z`hl<`%72C9*vt{AJaBmhwo-HeW*$> z1`~ml%kIY8jKg7@LYIGltQi;yujb9+Fy3h+2|dg@uq9GW>NlW)F{Rn#rd>V zaq{uQLFBgcO1e@ny?gbXJ}_c9*Z^Oe=nL<$az2dB!?={oU`+qQjg-T}yc%W3Zd_?- zc(NUeOdP^T?42m*-}J319IZbBgu!&CzXzcCnO0g<0BmTC7sq*xFx$TFz;Iy#BE;y<|3C&z{KFXfRcHGQFXcd%l^ z63U+Ut!>u$8U>$%qv#dV8L&7;zF0K*>VGgQjppK72$9@GFOELW?>JJ=g55L+jvzFX z9O>DnS=A=gksB!?Yp?_xqoIL{oHK@j!C$c^RR0ZYL8CA9F`|ViniyV4oRq>1mgr^I z)PkfZv_P3PAf(m3Xm=+l!uy`sVG(GOw;o+sYVaRisWIl06n+CK?Ka`4Ylx%C-hJup zA6$uf^$36~IiNT&)D3b*-Bs91pM%MY#Z$^FZ^6#M6oVIahlERLlNV}87{-P>B^TtP zIP9DTSpb2V_I})=^j#H;zk$c@^mMp}qFN)-THN%e1BY5gx;>9m$XZ--RCSsdhLZA~+RKDiL705Sm%gP5>kTRik{Ah5J=<-5`&MH=v za2WjjRbfn}N2lBI#l6rGc;uF9q~{i9xQw+9q=*Bx&^*kjd7V>XmUFqGAINR~8{@?u zZ6$+PR!Qh85b1I$)3)DsYn2>StEF@YWqEWRcCEZD!|#vU4t-^!25fCjrI_u|4;US- zNco2(q$!GBzbVxXSzhSW6o&d$xx>>e8s8@i4DlMOWX0U ze7SUBX8LLvQjK{M*0vFXG&w&7p*2xI>)4v6oPoqb zM-D&0J-JXRf7k|X&FSQyIu?dZ&6?Lu!**U&>(sfaNz9<&NCs%R=fi%3;-=K z+|I}!HE%10o__6cX_YjkcEb@qb)40?#5;+Alxik<>(fWnUL?Xpl?(J1RFdsQRxB>{ z_K}h}8%;PI=v(RvmYEe*XQBOFsCsR|x++L_rvO!`C5N5RVqiPA+vW0uE<*h|UHSVi=> z@IZEd#%K^X=Gc_{cKsM&zCHvsFq%|^9W1X;?#M0>1nmV!E%$n$7pC z)?=ak$hHsIun%$*r21W3{XbbDs+oo8EGWS5P*?iBvDf$J`QOT zDvoOPArvlN8M&$~#GIX86g5LpQvP_PpqP+FXLu~ECA@^9Wn9n(#2`(B7ko6=7P%g& ztFBZ(N-1CPY*v9zC~4GQEL?j@h%-cJ;_(!FL+UuVIOc28IJ!XJjF>&mkU)bX`w$Cr zRgZm-ZIUStmakl-AwiZ@T>gVwZw?CU3l;|Jff*Y~cUjZiY?j)&YUy1rAjfU*e5F*d zWKx2pEq}OTv6pYH$ne1MfG%&_oD7xYMUyV+_Jh3+4@$#!Vn-X-!M}1PoV*ZgnJ(67 zgSApl+58q1_WVn#h`ohmvi$Xoc1SP&tVGqivV3|oH_YTS2};wm8@a|O)?@J>xJe1 zOIOlh{u20s{v#oh@GvW=f3-Ho9`W&s^j&TD|J0R6A1QJB1pm^N*xz*}7pKuA4oU={ z;0t@8P3b$ZNA9qG-#wyZLU@>@OA3C64A`NukR|364@jo!U45Mu80QV)9iV`Z30KA= zqA#aJch8&k#4kQ+bI`Kpr(k|Ai{x(F=NgwU`y2Z5(*O0bmU|4%{la_6k4Jdo!6x>5V07*}r1t(Ii&xoV!+! z^ayZXJnCk9N<+NNeZAW^vRX<&geHY$^-kQEd7@3>;lf2_0Jcb9bV_@&n(V`- zqt+4em$Y>bxkwnk8q1zJJFO4V(fIfFCB&7IM zbkyPLD?~6P;w>Qg!zkq$f+>kSUeJd-dy|2|LmxO&Ti;))f>0Kxfa4%9n>9-%w$*zZLn+=%eEzYrj$-C$)L(mIQ z935TTg*`bp#I8T#mn`>c*n;m}0p&N$)_?EG+jFd>lw? zB>xI(NH)w$d084LfyKw9jR~Y_|Cu@%r8Bt3nj!NGZzc=FuoPaR=}K}Fm1O{c6dfuZ4U>KIdY7$Rnrq~t>vPX!use1?HLE0=|}Hd zrgG>^IuGzGlGj(h?dgKK(nmT*igdf8mPbX45)u-)&2^4Y{-U&BlCpxofaF?s_%r9a zi=jJ*5@Ze4_|hnZiC=`P?-}zhH|_1uu9gAk8<-4_&oA-~2cpghq(FV897bGDNbGHt ziY0VvumYvLMw)0;cQNHO-VEirxc&(qU1iZC75N>I!puFv=+s|qhs?29bj6OTk4!D# zU6paUy3#ZvNS@uBN?MpidJ2D#>54IPrGL~0xff0o_mPYD8#1%_(m|<6FrBi?e25Li z?$4f#?ZnL(XpTvy^i52YNBNSHi^Cqog?)?x0Y6dxZ0EQ?rbXOK&F7LNz4(He@J#9# zwNxzvK4Ab8YYK#@4B)t>NP&0}PmEX3w~t9TKuPruP&Ab2ZbQ~_Vy9{Z8{f|MyO3T3 zEUV2zQuy#w_Dwch_Khv>H07@C7}0wczvc;P^4RN~Pw+}bRhy&1u0YW`1HpelvGiVU zqwd!u#@N0jE(TAhZ`Mlmo^{ULOLmYFzAGD<<22-Zel`4_}OYlv- zB}Ai3Zlg77Je;-L`FX-3H0Y8NSUjvP2+mRQv$5P%zsINIZRfF2CX#NN$Ym4ZB?m?e zzJv+|js3>FwRCZ}tUX#~oOU%$X70vcmjYW1U$*1@K8;?}?iK>;@DIe>wKfzX7Q0Yi zKPMv)d5Z1EqQ`+13F5?Y0N#Qfl33`!<2hPfpHS+KrCgYZcGOAPZ~6A<8Fy{;+>dc& zG0R8|G5+Gwy7_6H9l??nly$>-J)p?zJ2HOpD-7eCQX-H$XV>-Ekpyi@UwmBso;20T z3#3-kE*xI5e#aeuu@i(!nl+w^(NNe*k`OMPD0><>JGaQg))hLwG3=d7Rac7i5HIJK zbE5z~>g?5`%nStR_=^Q}u>)vZa@$JAO-qYcvX`FWP^p`qN|XCU+GF2eo}2xzJL?vz zvCIiA839ySV8Dbbl}mi~Ruk$eXIU;E9b5a%)LD)78nm|mS(b#z$GfH1dG z%T`^eZB@e-SsI_e#(sypW&buOQ#i}-{fHs?u|&qqjXN}fn7s9d7voXSxB)P9fR}yaW@((JYBY8cA zXWebAw_5`=as9jh_pcXXfd4#wA9fbIG#eJ;pbxcvO`RdtTeVRH3 zkYCl1{po7_*A*GZDO24gVK9DXRBQMxm#Te6p%e((?_jJ?7(YK}G|!PT4$fXrQfzOj zh~kPag3jHcjaDS=NO43kQQo-L8j%EJ%Kj8=d`&Jq3_W)wB7sm%l^v;+7W_0J)+|>u zVzi-&_fmQivHub-d` z3bY;81~~b$m*d6f0J=8(l_aB94??)^sTU9BHuXA_TkxQ7q`&tigPL5Z5(^YnO6lY1 zV~j7j^GO7yOsq^_;)1!yIK75jZ!%hs!o2p^+}J!2^@HeoftPmR@v}S;M}NR-mZ|Q# z?tI#-)Q(M!8Ca7KMTCUUI-nKTc%fnkc6Vu6pubT2*gGrSIA-jkBD z_#(!VN-p7m20EsmfOERZVK-Le(xg{E-klY|-%1vH%uN3y_PR#8DWl>}$RQe6WSC82susX5+^ zi%v+QBb6Og%TF1s?E$iSFa=1pym{Okovd@A=CfkK&3rrOrB`<_7Z-%rgsv4QQs%Ta z99E?YTy059uj;NL@rk1 zuJ^Qs#HjJJv%oFRs7f}Ym2!-ewg0w+avJxQqu@KPXYC-nKSjkTec(vZp)pumMBG=+ zDnqFK#M{=?60nBEtq_PdM+hp69DN7g^_X_TqPrq+XC)iZE@V}y1B4Bm{{?D5mA`AH zFqGnyQOM*&%JRK9!mKc(R`e?@+ex;J!UnStr9I?z+gt7J_KspndC5ejN?DO-jW@c3 z{`ex`5}*jJ=#&*!z>>?by@Jc5IcG+wu#|fDUIMc9SfixdQ0pf{+^7 z>MEv#&erDEW@mG!v$@rYMx3vK2rbOdsh29j{^|Zl{J7(yTtAj zcN_f2B|k+X>6XF?z|B``ieAa`mq7H-Ai`-&X6Mz3b724Z(u_*@a|>!AxN>1224>}; zb?1MQ3CZq+g{CEa74de)j9ian5xJF9M@|i7s3?Ui8TdgRTy3Sp=6tNVq~(CWkc;;U z2RB3{xmz*w4oZ5?Vqdv|mrIEKH)g~yfRJj0tUw&c+uz+?S5Vmuk-=k0D&d4qk_+Rv zDqaG$B?w)i0q~`Jml(P=kA5C&Ayv=ziefZfQ{J)}T|I)4+!EfW7w4SK2@Q!7 zJfzHyFo+hQ#hyXiVlY2Ovd)Mmoz_aLl-^VDg_k88>iPTB`cG@wcQ}ud`#1@WOq{@o zS;;&3Il=`fWjtR60j&|vYGdh?rfAIiBR~_4gDQ&Sq_d)I_W@j-P5XwKu0)>rE_yaPE>Kx`xNxYAY@JVn$r8-G zi$b3#U<8VFWI78L&`G5TPPFBBVSXMZB?gxjv~xscd?#TFj>AyKEw%ke3;S?v>PT9p zcIEX!s zy%HRzdM#b2qVQGeji6upt*GVGZ1CzIr{`}#g8irZ2!=#{NJltIUVWh(qVy&r<%am| z`Zx^vk^4%@Pn!_%W3$Jl4pnX}ZTb9h7OQePG)ND`7RApg%4Q?Ye^0Jjif-%aEyNpG z2;YlD??ohDCGbQ^W!)BGlVreHbt9*pNPtiP3uw=&>;;zJN&%*DK=rwxDzPNs6@RNi zStp73o3mEXx!D>SrTLv7Mwu_#71fES^crrzWdD`-E)(TL3_ z?CN%N35M8=Mt8*(Q#uLUOfAc4h6lvp^+K&w5d!{O8W#Bl&j}8H$fBH_CLk}e=r_WT zxHm$3UF89y3gCPwtwMlHQIm$E#1}jT1g8@MfeNpR#RP%y8YS`uxK(33#E&>|pSX=i z;}IG&-a1Vr$AU&AjK-tFo%_9v)2Ql#>qV46UEv;bMe_HA2{Gfi@o9=Y4?uy>ZM1unmh_}1#t|yeMT2DOncF~4TyP0X< zGCqA`vPAjwnF&}xN9hHGB-T%QY=;m_8J`1cGm_MT>8 z53xL>$sm=#dEaGAG})}Ox`Fzz#@nL#o=s(o032=ttJ<~h8+dcykvj>cFL#iF%J^ac zx`Gabuf9s`F7_n3CBayRg_1iOUEP@%b5gm=VJ0LL9<~CTKTp6)fFzTjMU1S36Gj<; z2(4K>^c7Bi!%+~^aR@p^P}oGtFCbg$8Y!|;<`n)B{B+94| z5+^u|L?}8d#wK46BI00~Ph45y4H=OE)Cv#;TgR;5=QH>Nc%Rb*?fr0p4!^tT@w(l% zj^LJ{2#-)kMSMoM!tj0|S15vgncsObNQ9IG5RJm1O>S8G3$&I{MgDSQpeVI~8qN)o_(16Xm)=!S|Gjku(8akjUAr~>$xwduZv zbC(HTp%&*&z%87ZIOJ&@j+sv|%l&T2-Z2V0|DN*n*!arpAx~IK_w1lyJq>@a0d{Xg zLbTmRIL~naw>2V3nYb&_y!^Tx8Sevd&X<4x{Pg&+zjuDJ zh#9GFa(*h7`DZdE&0S7MSKZJ-Pq6-Vn*njrk9{;oT{6+*S!njNaAs8#Qu=x7| zfd*w%akv0Q(g9!H>6;;@jpfi%5=^0ZxgN(lzMF^#%0$KlamQ0y09B(9VPu<#EIu=w z%qG4Q@?MQrm_*M5UrXKv7lGX*3UsJ=@BZF{fB<)^T&$DXYOdhKO-UBtA;JO(w85y3 zLVl;S0>L$Fnh_Z*?pqR%DjQ%(;!&Fo#nKuW+h#&FXR(Tw^4w`LWi-AwRxIl4p174pTLO#lpNHcN-$65l#@T z8k)X0LPyEjPzI%nfmH&rs?FPkNp#Q|Yne$cUZJ6tDmlp&Ku&#f9o_cizv7+h#<0q_ zP#VxAx)ug@p9NMmIe8RfpNxQSZFg2QNF*bu zoTNA@P`lN1rC!cYA{z5$HaPIQq)Ec%H^X~@`LZCKW2#GEci9pe9%-R+EZp$*N;K|j zOrtEYt`eIESdo^JWtkbjOnej5t`f{FV#e*@d`fnfa-)gtn{Mk$CHqO+P(yKEXvQjy zl#D)0cBMi;3vW5$m!cP11DX0g=aJVP3^AxTWad1UXXApw<6IK3W9!!7@O$QSMl07I zO7`gHWk{RGCNqc;<6c~f$!}>$dT3{Bv$!xu{zI2IH0Sxz(J zM+p}mMUN)d4U8-8T@{}Ji}dKjESiw4)OOS9*on zP-{MECbJ$SkTF8t7qnv45VWHu!IZKbl05eG9Fd@aT4`gI;lmkbGFDzZ9&o8dC#{lc zq*To1IAUPu+>!;21rkPsEDDDH)%fZvCcYaEU%IkyP=lxQ`wOjuOMuPv^g+6T!le~SyC2K zyRu01r3@D2?SAFGj>FXJm{!fkpMpZE@+Q1-$A||gU5Fi-Qbvr zT(4_oL)i7lts~ViSfRU@FABFvH ziT%u@oG$>~5wBk=!kO1dxEAe|K85PG-(reK~+d&i+#^}Cy!tB!JI{!+8&OSdH+&b{aFEv)Q5PCboyfRQNmyG3qK!R%B(C`o zPe#IVKxTk4#;ks@q6h%%EK|&VT}PsYexKZ?S-)RXXxzAPK=R56lNs>}jaGcG!dRDC%Wme}qfx%zq_Em6FT1SB$ju#s*CmX)A_`+HBn!uR zb%D#Ycg{`77u!IbL10et?|=W>QHqfX$-oji$f7)wUJ<22(whUK^2%5cu2L#O)JXE_dTTp*1{Bb?od*p}6meSeRBzQ5O@c~zu`_J}Zs2gE~d^ziUAdUz-u z?S*6T;}3A@nk*ECelWyI0`8wHE!~S^3-t?=DJ)9ae2FmrRbOqs3%*(06};)HS1U4F zmu8`8Ht0v8<~orGdS!eReZIv}f=;tRyWk4@VU;a37hA!j3EN}NV&S9@@bu-oe<{C< zfR0SQtZ$6Hj%SwyNgFLZns--z(DLT^=M-}2D2*y81moyRJCqj{I{#2G0Ji=qcz^j- ziLZtTAirq9r$_?2TxlJw$_ZFCrt8R@O2w?LuG))48VQ&^6-L~gV;v1b zPjOo4OlwdS>HrDNvnfniX$ChaC#RQ(J#;B#*P=0zjGoU62)07iUc?$fRoquDK1`9b zGV?}HDbZLiXZ(?}z^Xxi~#h zDwi?K>Bvj)`DM-&C_a-kesibEkj=;Hi!AnV3SPgNPGsPWEY55 zG+<-HcM1#EPEJ^ELW+dq8P=||dH)JCvb!^XFaFi;PIFHPUG47hQyoqzLt9k_VG!jp zzFGhTQPGA%yt|w>>L|Y&2R9~HGW)<)UAto==V3#~__uLJ7U&e}l{&;M3M_o%ARuY} zEy03ZebM=1wIxhJxZ$hsko2+$SHhwXywaS*`lTw<-Ki4G_>gho;LE!uk++>`{qGjgAnB08pQxSy!-Yj%WmiGYNNa8ubt2)qah z53e5bI4^sR*7!*(RZNJwpcLmz}tA&QWg5%ln|y2}4fLiF(9 z@rhL@^q%c+Cly|*Z*9m~T*%5LM77;ApWhZZ_7vme1OT#cj8&%b0A1sN*B@swob&w5 z<+~`O@0mPnr!hx@Y$N5l3gXpY?fhi}v zC@BKHK_tszTYTxs4$ICof|cW_=(A{$^tm)+eqsSl`MMc1lJx-ejf~*1N3(&dJkE#x zx;XkV9Yjfo0>nC5v#bcG*=#jIBiolZyo&p@tKu|j8`XsczUYTb&=EupDn?7fKFtO= zktsa&N8#eg1r7U!cD!3b#|L|7PqoT49ebk^d8U-MnQxE&ki;n$ zCGs@3IA;N?>8r#sqz^YEJGQVRQe#zQ>-N#Lv92&nNEZTGy2o^s{K~XC3KLaFmE% ziw0okf3~e7;r3J;>nPhD>8h%hlI}tM{NT%_HX+#+Wuza|!GI6!RpDQT5c=^)8RhS% zpNtz~^a^>SJD4ST8>EaHAPUnfw7b14UyK>)GZw36rBK6nQ$LRxdS%QBLX5?Ib`z!j z3FN;meZ7j3upcJ8^Klfu`Y*OY$FDebVH;uaYIH}l0slh;WU)5K#oJ5A<|wWAjhPlZ z8|6>4&Ur&E{=eSSElA#6^Mai2Hs$fZyt2!br+!K6-6$jhR_@jTjUm;WWpk*k&6MM& zP_T-NV7_9RjCBLbwooQN&4*-EBw5V}uboE|BGnR1F!(ZiKF(20Fw6C3d#PhdNdw8% z+{o)W5QdS$y9Q@?L~@cbhv2_q-lIZOhP2Pb$a{16Ro+DJqd^vhJ_NEC^$X`p$Xn7a zxmHdKFItH;Vh0;3hYu;d6~V!E1y$mWh#D^|6<%)EdpG`$LHkw^Cli-36N$1#WH^x` z8&2z+C`FvS03A`B0feX-LvFVSoM>3JC4CBuE&eWvejSt7QA|VtC_S=Lq-7Z+&5E9^ z6gtOrfMPNsv37F<+w?d_axD}w*l0o!z3!od!*AZb*+54puTMA74}0e)8|d)-{PcVS z?VlfA9_{ZPx0UZ77!~t^&{s?c$GoeadId^lR`0!SS|GVC@wKxiw1mv{TLgaOEgevh zUNe$r6g=W^%&~Nbt8NUPblY%&@oG$VVWoio*p5juB8 zQWa&1Um-sLHQ1$$aCRe!xR3>TMQyIshk~+A!QavA*&7!r+KYC-mtk!^*`i(pi%hi| zX?bq>%cJXp!iq$Oa)JbVzzEKQp3jeg*>_QT%g{=yQ3b@|gQFhs*>z|T`DO~ug-LEF2VdgN%WJ6wm&aZRw8bdH*YCcZQR{_3J(AiOE@*(|gV zBA_k=Jdz5918Z4)+0aZ7A2qnZ$?z8xbeYBGxuat=#1n!rlE(Xpj&nR9a7w|wB%kXP z^LldIClsAuTy|a`oouL}oHFvzkiHiNanPm8;ftf3DW}+ssMKnID|)b}z=`)=Mgp|P zqTsLV4fxo#QFfSwa6>J&SCQ3MxD|pKpP=e;jqx|{&W~lB&0ePiNaGAYgz^D9JGc+C zufdeHQAf0(Xx`oxqwzQ|GXJw7;Uo!i(Fx_OG_r&xCrzW!0L~EJkPP-ADi~XyAK2$m z!amAW{HH|wq8BAWpn?#fIYnaR$(=gWl(yDz^XCgmO~gTwds={~=muus&TU4o5_R~i zJ#q5Z9kzB$wxu+jt+#oGEgNVAK^}7^N!M>OU$e5Jv>Q|rg=EVDZ$+RJIJ846it}Un z149e`Y4Hdb`--!lhjB25GXwWvn!3!xL6my=-Pi~rUMU?ysSrYwM%(i_2L7vBC-^DLQbV7$ZdSYaKijXJMEqJd(F4TwxdI zFP8lFlHYFRx3xupol*ym(wGlNU5v^gNH*J4Trh{`Rj1`CqpgbM$;_!gsHkLd={6}y zlVW>xQ#2by;5lR{wTl>0vKSN+_NQgupg4_8W=}&RY_MEy8b)QRRS0}k8kVK=1A{YF z+O#+my6_-}6i^7uZhZ*fEi=@ORZ~_yb2q~s9fXI4`*^-8H080`8^}<>p?r6wBCYXy z@1l3=BHz`jQc24IE|BrYuvpWyRx}UFZ%Ql!ug zJyxbL$klv8g>hKSud_7i`s|8Jv(`-NOUujE0JxD`(D;~#T~L;nPrxcDfB8>K0XdZ= zP+-r%4v3<2vI1uNp8*>n;{hyLfF%pCWC4~efMNkmJN$)@e-$R=kw?EIQH$#mokw%i z$qdv+o`WJ=L|`AktQtZ-^1iiLlmJQOYv|G!^c(1trd90nXqQWiE2w z)Zk9riys!=S5VD&G>racWZtf@O7g~uiCa88&M6CU93_KV)@l?b=T;?%7F4`?39m#N z=KZsGJ>>0fdd54MBuudl@)|Bv$)^iPiIwu0jwJ4s|AgGi4ek^wa-sUyZm_AB91kSS zgC5#$cQ@NFn~Jj`Mr$gVw4hvO@+)_(!f68vnHjS7;`jJY_#BY>q%TZ3tdHdzduPW`lJv}(=sRaBg(Hc5>b8>oq*yoU3^nt&-uriQ6PZ9~}>LaI- zkSWNT5=lT2JwHY=3H`%PC!t?k2_=lT4JjpRZ#Av75=gqnRufAg6&R!@wS@j&PA=hC z*~S#vQKeg`^g@`@NngXJRxE!ahli1(hV^Y2^HXU}f4guwEdtqd2^8=J|tQyT77GG#Vl>08QjB=g+M%;n{*_w|+2ETSBH zB2E|YUcNAreu7Phk+Hk^2rEvls!Oy379Dn)X5yR|M3rW8p{;UqusTCs@BzHK0|}(a z9F>H|Xm!&)^ws89UE_2+vtI%mAb1;`@>P`6fW|U6zm#3vu(a)5kkVNZ`0|M55hNHB zjp=0EZFG1m8Cs7^RbC#X;tVkQmOD=N@|Gr%4AQ7lR|*CXqYJA9+CM+wgjV?sg8`!v z@JGjcR4qAz18~wwS;vv}N>-|4Ib`^!5y`VCP*%VNFV{mO0VJ0((3&Yvj4N_Ih%gZ2 zm=8xxDI6~Ag<^wg5?VwX?T#d8J|uiK@!^SSk7ZyKU55o3PDvr)ZEBf=w++F|Op6*) z98@6<+Cdhwc1R{B;v5)*)5?gn+bK}c7I=o5_9S*D-G5Cs+ud$^bA=yNzF4y&d91OB zRW?}(C4y+m#;30PwFu;NbS=ii(W@dWM2WI;lBs98bfI#bKps>k_9t5f!JTFq3G%pz zA0x$g9LHBUxX~G%ix7X3!z)&~oS+qSl%N1JX${^7*-R!Qtb$^tTPd;Xs`H|=3p=HR zWqwy7&RLGAHklXH`ApL$4Q!$Xk5U7YPv^)2U~dRXTSXMDiUfj`m@hd>o7!S@Vtxg2 z4y3LqxuWAFM0tjSQL6%Z*-h5+TEe-%~JX59PU!_ zt24KN6(DGCOK4pqgSHWBOj9wD&!9vCiv=xcg-l2fkX*=g(rVW{x7RfnT4_jBvS_#) zL9(^#8R;xl}R#wozzy-mkKRSw(Bnhy&BbsnZXA=j#aoQroh}>VA zL_`c-M=^+4hfRb zLa=U~n!SyF0R6a#Fd%b)13}}&tB5Zo&o|MU)gMT_pr=(@+9GAFK?$>5>DuICfPMw% zHZWrniO6^65QG&mPVSI46l({ZiPG00S(Ynclk>w0@-Cx} ztO!M=D6R&cD$R)4FXoI*?V8;3(=y5n=PX6D(4Lvv3)mL@2~@PAVtgmrYU6@NU`}2N z$<-JrgLWC7s-?K-1_&aCMiGNlIW$v?<^Dmg6uBJEXegB+7~IhW7i$i3)ws-R>BJdO zWhC+ngY!?2MLEeLEbh%{qI4^;$RKPJr83~k26z%>1j}2mK@kULWd(gR&LP4m7kSwR zdQU`}5u>1DqF}|U+b=O5i97*Pfzk~zGRVlt_sV->IWvilE0ZK=rtBjJ5fMponOTc1 z<%a+_b&$JTPdYGwdZj~3XcehdOAxQNc&| zplcC1eswH;bd+q!i>My@Q8o9|gC>CW@JaE1rRje}Y-}aMxwn=b6%aPJ{ILOHhg`c@ zY4TLdinqfm&YH1;B(1VSaza2$b*&C7$w;Iq&$V`~Ex^hw1Gf zYzHS7a5%}2N;e7hlnQi3$7=zCFSo54TexMCx&2VohqGw(=u4 zxm-LyHSFG&mbAEh`W|N?gu%yo1ggk8&SV+;1P#~Y_=4m`@b7))pdH>NcoYR_gp=r+OFL*1VKsP> z(Cida_nP^t%oygLp|uxB4QGHST@_^RCQ8B{QjEieigX9mm*m*a@(ArMf028kB>v87XSK-pEL<2I3BrkBLzPDDmAVaeJgdP)o zLqe666U+ldghr$?|LV)wQuIa{hHXfK8wR(#0wpWTzCooYCbubNMXW%5>0k!F(-#G8 zbd;O=o~t``tKV*|w77T(A>xX-zd5xU;@l@4%TA@Gz(@o{H0(tr1Q`$-r+yIpl$q#B)ff6~|E$4U1H2+0E^0w43G>0;|w8ol}j#=mA^z!hco!{nv zneW-$+}zyV*@1sIH#f`wzTAEJ1ZN)=_ z6;w+^1Fs7N{s7k;(x&VxN8dIr(wJUFNvB{4V2c0tH&uiUb>VEBwB=7ZkOPwAC}zT_fupAOkZKD3$}9PNp#bZ7 znic9DDZ;+TcMJs?X2Uj&`ZdVe<6MQ2QA=jSu#-gZ&X3m^7%fWt-rxHu!^wcugmM>i zj#TtozL@npUROkPZ^54;&^!Oe$xCkRk$2}{0XvCUlpCZ(O7e2P?1cGXL)UA`ZlY^- zUC)qrLPK&!v)rp}EC?8&z~g6dmZOu?gTwyW>G|cW7i%WfnwMOaNDOBSg%tMV=1=WJ zg7n}OcMsEd>3S8=qq7p2yk+)SV56Lj08WV;FsXJppZh3%SY5Yv_=^JIUft2^(aD=0IxD&cQw||Ag@66)Ux5(FU|Wm`&IweAaI1?@sYU&m zXH`mtxiiT|Q395QsNpeNe9BQ9^ZDfb|2*@GdvFH0NSIRvA;=lD`9cKo z@4Z58ux`}vGfH!KEt~cQ`O@@8x*`mPvfGjn8LWo49TnU z(XHk5KU_YSbv~ek&u)^Vt#!FKtE*ymSPNI7M?zW$(Ef=3sg<<0}24ClhaNPTnzFA{T4JCv@xD0 z^=cviKFgz;U*)k2K+R*N2Zs3PkiJj2OjHDAW^Fc5ycC*buCybL5lp zbwPWYLAO;a^9>u76f<%$rFNOoO&Dd!Pl0KdiF4av{BjzK6s|U+?NVmqQBltS7I}N{7V?kB(wDTurASZqRz6eAI((L#ZI>z+ z^R3~M^&lUg(Xh<#Cmm44*P<6ydgkVhbg#9Yd;?{SUhGP<_k~#$)b|2kjZS3gnVUCK z<`$itogaO_cX{Yvo_=?DV$O$Mx~3D}!oFBHzPLOtFM_YR48}fPmQ|0~Z}fYzpc z5vFH+m-k*F&y$)Mb2o+j%bMWKT-(~KNfYj^mnn?K3i*|{3(ARNtD9vMOs}Avix@St zKLzIbPL)->0>k31KuJl{EWBRI+I5^pW?X1vu7F;khlh?bpyyvr2-JIy+M=F*%U?)P_wGHBRW=c7wgF=c zcX7`bR;TTs9BHDyT;Xz2$Y1tiXv$L0D;Yb-Ts8%l1KF58IiOjjveKWv;Cn3LzKG7t z8(%=yFA+hk6F0&~(x>i1#7~*IRcCGE27i0A=t&+vJxyyGqGZoCZ;eYqc`7!ys24y*x#cKa zNj`!kIW)*eb?;aAu{35gm3FWZa1x?5b2pG=(th*0e{s0~?)>QTKmCKFi@k4-5Boe8gzur6EIxMENo4OJATh2cqAg<1Nu zM7=~ZW!q%t;i08d2=|oU%Bmm_fEV?kSJ&0=t*>4_Py5Vu|6^E?&eKQp{m<6+=I&1U z{%7mUt>yjCb37~PT&w`=zz}gJ`tscAXJinuJiD_F4Mj=@F}=cZJHUA!leVZSqhXxO zdb(PgMM-{*y#HbyZ=nVh8M2PL74!qdt6}J3oTfC(CFiT&O5-Q#3{8sQjwQG zO2)UszGkDviwKK&@!Ie&RWTU2Y{v@6FXas>-Nq>+W{WLgtStn?SMU=x99o&cue1;M zBtjzEp^Qrc6B%g{>hc%l`kDlJ4|yl_LIS{F7d`~u;jd#Hd-aTg0S1#^3)UPnGMbPK ztok#8ynl4t-5r0^L+yPd#7X>8O`hfCmoCIih8~%(&7vt;e=oWdtS5-`RzGxMG5GtMBIF9cSpE{8U9j7V4 z-ERFrMB7k4=RJU;2wjl?j~M~iJkzyN*pEgemiYp~{amfmh@my<5hQI*AyIrvg7tRm z=o)3h&P6n$ZF)r140spiRsc+hH3z^;XloH$m?=5viTbGVDT~y8p9SCgM#D!h0QTOi z*(bDsQDZ?gQZ@w=a9S_t1g;>E{(mvA)RyRU_T^Q31yd|KV>bB?QT{-f=>Rx-qX_rV zhFOJDo`cqlp_o_}e=9=q0Rmg#fFSR`J>>PhMYm9SgX_3GGWb+2=+&51&ZTJ=Ys++a zCHgSMK)t4|#8^S*g3y-f@{fuwrLzI|z1#FrUlNu-V&pL8(-7eKIq#(r6C935W5a5s zW*Z2nLgAKEP&*kU2rNvgpaLODfKh;G35Mlf5e`bmfE!(b!Dt!@*#;wZ8j8AgIlVVgh7h5)$fY+IF`+}4T?)M zMJ@KyR1pkqo<70@(#|nl)>o}ICrbTze~&m@a83pUVpQUQ?8WhMl#>j{$om%?;`SGw zqgl^0R7+Jj+~RT-5Rtww^snxO{y@!(bgD!y8z*pk6c~|slh@r1ngKD1+Dm@gG?wx1 zJAv2cpbaLoj%vze)*kUml}gRE@~pV1PrVN9#+R#d`q~m9Oclyzowlc2cjZ{Fk6wny zlnRen$N;?x!IPC*5x)R6Nzp*iP@m8A`9CFFf!aLp@*PAy6x`YoAn6t-}Ms0oL!)jA>ATy+SqRBtu!swIVHC_NrW}Q z(@x^Ie35VgG)8AMM0+w!H^~~Nb(rG1DV>5OBMNwDvc-phkU7lS?-C_Vr8!otDu4{M zC}L6?9z-k~!*$;`<8VM439O@ZEs0PD=Z$sj1u$o&Sr@H{q0^akJ7?8)%L`W*Zd$0i z&|Hf&y4FFuB8FHlobRU6m55zH(E`8ZSI5ERB12qeY~J_HU^1Z;X6l8GMLYx8s_s## zYB3C&p|!@L6eH&Y$+I!xG!Y2uqfsDcu8zchH5-XtpBIlQ&WD$G={)#5lU4rsczaE= z5zb|d(X(+Z!_BZZI!e+p1V~v6qktd&l!7$km5X|)eXpY$l|MeLPhqpIhBObUJ!(w* zY+;_Y7SJTXx%gDNlqt7}DsN2gkc|&i*LeB)mbu*v-R^hfj={F?y+S{? ztk=DkFO8>u1X=S6GvaL^Pm7vA83ouVPwPL*03y7AKwbZa+|?EViMH=-iA!g~YP_~H zC=!Plz=2RA^~DC14q_OT2vpXW3$j`eBEQd8Fvc?#Bnio)pilX)8Oqa}9M4b;|2EDD z4rc*C<{=E|X!g5cGcC?0(A7RVhT$_-h9a}e?}j` z>mRnCZ`nj^Lh7UsrM)bt<|v7#d4BpXbMXF#sne=bhPL&_XhU`%jI)d+`CFVqeFK_G zQF#v2B##ntVn7VyaybbYj6`~xO3=VPE|uoqx{k&hXGhDO=##~O6PjEvwP89{ti>d; z`s2KA7^|#HP;0-1g9#@Dy)*Hpc6atPbzWd&RL; ztC)CIJEVeZccYcgrgc?XekLo~+;sJw{pP;L^yc@rSzA6&_snJgnbL6X;2+cMKf60y zFJG4JKf60y%lIG9^3-TZO*u6O?_J0Q>@m*Sdqy~4kG~VYm?hp$3E^SomEeL?Inji4 zZ}3dv%9T3KZBS4(=MmS`*Q)8Po-moe;^CpCe^^+siZsC&CNlq(Y)dA?dc=q;iQ^EsnN_mMsUU>a zajwBFeM5vQhd;Vyd~swZ&1FLMSa#9@8G==sEPalWDBpuW*WJw+3Z2@v%rd;egW`8h zt6Diz+h$S1O3W)1AXd^~)^noR?6wKV(AJQ?PdL>2tJJe7O&%}g2fUOo91<0WqwLRx zv>`Wpt&Muyw2pB0FH9*i)41GVsGI7rAy&E}k)&3*((0y!%W7k%DpM_bc<4r z#fEt6hyYO2VAYwF=F@?ckU5MNXqMYfJ4?IM>0 zu^mAc{8nLw>R-3LP<4LBqLi446X-_| z;svpxClmc@=}p&%W~7fQ#mFuvZq0=g^$i%zCp1xmqW7_(WRsl?qTU8;u8 zS_j`-h1KpRY6{mhZ8ksZAvg;FB$@DG7w*S$Ntn3s5UGOtF{K^a85^Tf_2^2b)wst0Q42gf%Kg}{KSGg#nemR$P^=N(y*~&i5rx^}& zS%U?>Xyqx7c?I8WLWW&>t|PtNQO$v97ppzKU4SflXg>n7xesWok+aIFJlBy_S$QAr zIC882wVnqF&B$KNH_+W#Mr8_-3LHmoPEOAc`+LXdhkFPA>7SjS?jK%U93E6)0?Vm7 zSX66o4==wxt=2t0y14X8MUVHsIXrfaykTt(XTTG){P$SLcdhi@|9Gb(bwLkhRzQ!PT!sHHz3vxt6~~$Se-L0Ncp=Y`gKfRM==36h3b&r zogDq&cZU!J=49{f;e~UtP1|8jONlwQW{ zdy+-hcU4F(zB@YWUmjodzdt-bdi|e;SA8Q0K^CRCqwoFmqqEBex~%OmjwU1_jGbll zN+ZpB`BeO=>XDPtsM4kV_UZe4V!5ier;p0>H!6G-=j4l0(|PVj8Z>Q%0^3?7ZFYm$YdF=YjZAdczmgGa~Xl!{?UV>?PQ~ufT-OHQuXSpfY8*OR|9+B_n-NoVg zl*BmGa`jYzKR^7x&@P`!dkAGKR$;sO9l0x+ZyE-iYi5=iS0*6NY<`$EShj{$JdhWm zoPei2W+eL#Y;kpX=pqx4_hhIqQVl(IDx-yFO!e?N&X|g4;hZ%qu4wbjHL${*Se%oX zV#h%M3;x`73e^&B-q5@AV|QC_rs3>VE9T~8eSoRpf&gBtE>1Ee-8RB)1mWEa^pjm078|RcFTPo@P1WG!+L>lyT zGl?|L)iiC#d_1+?tWKd$W30wNjdN@JmR_Mtec#e6)NXAKWmy60Q)$bV&ZB7QJo@>l zxt`X2bbdXTgMd7NmTO9jkMgpeU-2|cYtukl>X(-KrKNu92@Jrj-S;F`U^el8F5Og# zb6G$+<>q}JujOcjZ0a#IUC-ex`*9UiQ;_|b^JGyK)icfBOiHR}oW1!i$fhEDX-4*E z)=+(fFd6msUDXH~b1Hn6K`)l-o@LOB1ynr+piiUdS;od##>Qyc7*DC%nX&0*^o&OY zQZrETXa$d>t)+Trsoq(tcb-ALW9+>rQSO+e|9Mn9B~E2-#g3cPd8}ff5vnPN%QU5c zv*gEA=uAQJ;%CQV%A9AJu^ALO&o*Q8D{-cvc&Wnq^Qv$@R;(# zlV{ zGy`8K0G~i$zgaBRNqc{Yv%Gy`WR`v+70GO>kp)yDQxpz`qQQVkjB+nczb6?J;S(I{ zk*83LJP~wrt4NB->b7pAgztckM!0BE{_N?Zc3$#QX})q~09nf~(6Nh3KMe?QHqiT( z|6SL&S#I%;t=+6cyI=fX+j-Vh_&VNx5;^kU8(-arbsXZ<)|Q$;@UPQBfHrbEiX8yx zZ%xdDBgOqb41a{vhAj8MlJj5HO!~I_Dikbo-d9SOQ}eIs{QU;sZHVK#;v6aSOT;>- z86MP-=n2j`adf5d>sic>wM(Uh9tP*wS#e%ar$1=Kx|0I zafq%68sa1*oT_3b6%6Q$P6g3|%=@CioE)10Xc^n~-mjd7Rd!|mgXd{MdF1L(>_mln zp%m|*e3=wY&$dWZ`Da}s8pz*!fjknBM=lQ)Pqwr=)awyWVSPO1@~Cueu9A65zt(4) zdtK=&B;g{)bFMDc_3yp79tX!$9lU3fIe5geyCChYJ9D3C{Ed;9>qLi-5CK{#B~ZQ2 zu`WC_dd{#n&2U}ppH;X}CuoYb>Jfm5DQtC&m(fCK3q>8kP?Hp$V zlJn1Iy)>ga4QSj$m-}b+y$&d2QEGJT=RsQ6`QJDHzVrAF#^JmjygLvb7^Y{cr?b<8 z{?VDP@HDSPs0MmqdhP(Pvo|dP;xgg$No;}SL zo~dW|ghCa^ek|?8GT2YHdLIjV%~D)o5yBPTIY~m2k*uCoeDk`0et2?lcz$&9Mp=V6 zEP94{p7uJO>8s6$e*H{!TxU=Z){mk})otLD8<03eMPc6P1;%pPznzba8Zga&+=$u`UdeXqKxg z<9z)de_Hu)rUG1rQyClsurKPI7XUIatTF&=v@x;`;Yl*#1C}&$&Hf^`ggWzww%Y`6#dB(FQ(p5&o*e+O{iZmUZ4HYr&+N)pKafB z+VsWIH3{y5n0!m=4J(Ev-&)-ml(ntwHx?S+6PgcJT2}@yLYW}@P~f+%qhJda(a!mw z2@S~wiAj*t%%&Z2l#FkE{@rJ?fMKn4A|Y>ibcnJ6bC(}TnO519tI@Rm^`8H=AV^>=DzEG=hP<+Fr6u7C1v8fAzCOtAuCv`Ru z@~iWEewPyDrIWxbt$5RnQ&-5v<^xPQ_C#+jT}d#$XwUzm9&LuJZWG+IfN5i!jMWA4#fTlIeK zLt~n;#WX{Una5`)cKk?VoX=Kcx}Ca zHr<$ceA+kmcN*K25X5!%A2W~72*CCnBb>>QM4_VK8!F7UWyH`Z3RGI*k334>(ohyM zUOW#AE)V8*n$cfKkUOFiSwqytTq=g5jf$?p>MLMTb1wmlT{qmVE=Q{`^J!p4v?%gjanE-;`Gpo`Y&aw-^}50~Xuv`knhJ6J91($m^iM4^ty zGF#PUn;Kn=E1J=4scwmsdaE?wpiQ%u;H;{rin>nz`Ik@d$#4=3N#>7e66G|Dl0hdJ z;w)#{_Ae)+G{!k$ogf~woMah|30Qf~+Q;VR=H~9s4*a{hxmo^qYqPuim+sD&FL%4$ zot>9kf7#sH?ta<*3)=j+Ll@7Od)%b!hwM zE7p!^XVPtrNRBys$;lJM-ZFfCEBv!4V5?qtYKQ{X>F^~fFkTd0I8O8(2SuBmr7!lI z3Rvif66O%2rq9Vtoc}bgdAg*U@xFR3*WK<6;D21$_<4cT=uJk)DeIvhJ?|&9f;oN7!mI%sUva5i-qi|>T?oU#y&&L% zPODiZ$gILA_63QawE9TKhH+wxqfA=3R5`MW@~MHTD=|GY*RW;Ag_x!ih)RHtwey{MykDoK(5KzZNDMPEVx~?hSt#sbhW!flE55Hag~f2qSxSazivj*AB{u5kv2zI7T#y@6daiK|F*zI!4$0Z}<*P!Rp0fh{S5l zDT;FQ2D_r*V>u_F2Q%Wi6rZLM(E`bJC`KO~+DQwwCN7OYRPJF+2i=O7@;L!+u5G@b zo58MM)ByNmpw}&)--q*<|NI}})Ny;=#1Bk4|2MzfF6)1Hw_mnZEybH_Q8f>*bd_ z%l-cxPf3n1^XT(!fEdRgU zeYw5d|IhK9~;;`~BS<%lOB22FfidQJl z#)k4(Iv!W0H;AU&Gbh5;zrxE0-_Yeq0go4$(GjrPIt;@s(x4c*Rw@x&6EcGgd0%92 zDI-c?sIH>#`{vJ2RyGOX~g_2yKb)uSFndUVXkgw zM|8ZW4-QxgKDaIbaXK1X?Ks?l`MK~kfe6j%5Pm*2#xh{4IRm0glYKv9oSq6CK1n8p z1FzbC?q=p&e>dYR66A54|C=QwCtx%sU6gzsXtXX7R~_BhD<3`!6AyLv)Y7+vg%pLzEGkl-*TjQJ{B<%|Si%KqIwOeGDicgc-1S)^jZ`!WhA|*OvzYO?8qy!>x>7M> z(oFj3tcOl$LMmeR+(M}AicgBR6^DrOI`;ou5%|+0CjOEM{D{w7^gqROT7HOU3jN>S z+1f18|L*peOaAX!p2yMu2GwDiB)iBaUr>xctAq`N(O~LCo#c{wAE2tTQN#ljuSa(T}?^; z>L>urUP2-j9n#jNp0{r_n~!7#lRSls#pDbJQBnLmGzfD{_9#luUFvCbl^SJaC44)* zQc*@$B6$`C%(_cHQks~LB7NHfo$q?zRrhHw`|XyW z@#??SaibxCU3M0;0C$Z=j0DOpJ*~+=Il<5r1A^%@_zBw+%s$p;>jJx--_h2f@v?uu z%l+m7e!NT3hJINvB;hzFnSE_*y%rwM?fL{e(|f7Jk1)Ebe0z%if^in*cl$KS$*mnl z-2AX`;LI6d7*dw+#Sv!2m`>y6$Lu@JA_~KdF=IBmE1#ysou1kk{A#D;IIiUKflp_} z7kkL^deB_QD%Y@GecLfyNxW!nYXEct2y9Sn^SmmN9ntk>HIvU|}&+?_6(eb_# zl5~ca`0}=QaKnJh+Xt0%C|5U36G4Zxf@OmrHNd(Q%kGxk36~{=j;zS2E@hN>hIyX8 zu~iR9=VR!U-_8c2d$30SWE*+kUS6K{&rZ)TOTA6Y4C%m#BWvH3MJF41E2QsgDoz{J z!xON8dwY0!ezbq#hF8R3nYN?qy>#kCL08s78@h(vg$9J>42A_fQ}Y<^IycFq2llw} zh;kE`N7WQC8xCu|D=+FeB-F$up?YqmyEOxJ6C=HSdD9rV-#hszEulnUuB0f+N9i@2 zeejiRY2vBmOnclorvOn+WCp`C^QC=6#?VZDCMuXI(2QQ2qftKsf=aqE{+Maa9y59) zCz)DSGc6n|koTWhY5RoFBlN#Bn*&U{|9`n%*8lEoE#rSb&vRdm|Ke5x&#Wr^gf9M{ zSd+Y%9e_)C{iB%$I4|%QpQXAB{Q9*w^L2hh6Z2f|*EjPAx$2*9fap0cmKOwRRx^NU z&40#50doxPlbFfH4F>94p51(4hBp2<#srVx{}PhFr`e64f~Kv36`bb(|MJVS|NrjG z?&gyJdyeOk{GVI{LfP7mqfwL}C4-DG?K@<$46-!eN=00E!8@Fo4Lt?3=SKpLMVP+I zT7DSubvW@Etg`unb)@$kOs6o z=`(acx93H7r>dHqEeb&6%-1#R@P=;Eqgez3oD1bLQHWvI6`OG!&v1)=Uy4)ow3O`d z(;#Hzx#$?N$yiSRj702EVVf8vYz5&srteXhFyQjzv1WK{soTqKC&hYqa5igEMch_U zj^i5C@2CCDHuEWrIiX|1*05PvQT!HaE-s|I6;qGXC$gJdZpz z)f%YL{XUH!?q_8WtT6KN?1EJfAHhDj>b*N7EZUt|G;Men7d9`}7>#4=5mqd7ScI6S zb&m5+pN$KOCDSf+)AUSsYuxD;FB`4s^ZPuHKmTRT#r1#N-R-jf z_wLKxrT*_(o&~Mb$^vn*eA)rc2xaYnjymPzUpDkaT);e_YnDJzRGWbS8|L|1F^@DW zN8wy-iACj~pGQ%{T9{Q)Yy(NcltzgP^Rb|v=6pa$Nlvl}*2k&(ZJ(r$tE1VwU_pjc zav6;X9qXgsdfC|7kS6K~?m=jBcq>d>ry)myz)KfB56LWiQUN|Mzu3_8n2E6bs=9vB zn^D#BD9mRB)?}0n^s+QwEd;_Z^a|!edaabK9!k{3doCX`RHs{k{y1>DJAV$MGRH&% zFKcQ!%r|q25lgo6vwP;^|AgXx?ifH*_`jE%yF2ClPurWz_&?9`l$j_w#fsHav3_32 z4|{2J#4d9Fu{K4lY9o>bU8!MJdl%yrGy(EXsBnav);O!FI7??|I3Hf#rL%XUDmWYG zY^nmek%!eaBnbuug5j88X)DUGgCncy#UMch2EkQW4bW8qzUfMEDaWfYS9#-W+3VkR z9;s?I)}88#*eL-Ns9cPle&R6xYA8>t$y4Z=Nq1M{$WnAA98_f5A1=noQZE{k`#3px zLk&E2YnI0nl?y;8<+nE9+E`V==Qrap*Bpz*sFVZMHoMYi<5;j*HH2_TS zmD1LM{HN(1TXhT?=6PD&Vevg_U1{kto13%Z&*Hj&+N>6|_Qwj2>p~(y_U#!1&V^p1+)e$4`O%TI}P7eO71$X51yuwlPgYyp-+!kk35w zpR{_KYVTWqrs@B;b}RY6UhZry<-g~6N^DQbT+e;0|52^qJ^{x66VKmOFBOF zMkOoVX&y19#b9d&bbE(2%`I#zIBVdoxga&I`R!IU}am1#W8J`ZqqmlBDfO+`o^jO2xhRxoMASdoaLg} zn33GNEF&`-lOD4`Pvz?7M3rRyG@}7wj4(l5@HnSm!*`Np zH0#H7U{Rw^NG2VYhjg5y_c%*L*)$4K_32D-)(LPhBw|;H|F_d9{JJH&n2;oA=w4Ag z2~t3lgam>e>~3!Dh&~=fonfA*x`sl8?2qJu>}eE@M4)~4wJaLbLI0X&Bb*Cj9VKY> z#}|1tBK?>KI7X1-0q0}(6MB)65zR>-hhc_%{d3Gn#((@eCM-v5FE}Xuj9in9Wb3PP zOu*6H-l;+Kb7>hNp6=$?Rw+)_OGS98uS&%jNgTyq+J&yh*Vlv-CuckpkoySX_+QG$a!nvhrB5RLF{pGCisSKZBl zM#l)JNT6z*4l*1P^rDzx{Z6j|9e7ZF`C5NiT}44mnSS%2df-eD$&4mqNVQN5?yJoL zp$OrCFZg0*1BUe1!EaG&;V=2x`qm#sNk1mZARnUcP3c=eM`<5$hPFzdU@?l{Il0X{ zY0Q@(P9c~AM@8osJi#o;qBOT&-}=S}s2VV4ml(z3gHaId89KkDWFY?>MA!1KG#Qj& zo8auOZn$B@K!J%6^4?w?9ikt*{_aoz+x85csYROihXjWtL*G)CqZdQUa-$UAW*{GQ za2gq@WsQkbO{JI#T!ALG(}5id7Y5s#o5oZ=)V0nC&Vu0rEgKcs_^N*$#W~632)G_k!f?FLC_%eNU~GyQc$!Bj^PyA+T29jn_H+GPAj>+*+tu%Q;Xr`4!s=JmAbkU zOG@BGybEZOXEZje;4oAu)X%^DGvEyNVVEz*Da$j0N3s!d7)@&f>iYYlL*%97t2hcg zv(}tjpi{R8ie+88IunsPbhfgXUg5YM;5?5DmxmB}(2Z9L6h5A5bE){%IE zF-^nd!^096P)9$v&w&{zpX_TU86O{X7P%I1BCuR)c)ds+iU;R`-Oc7 z82j)`I{3iPw-1Bp`a_xwK12ci5R9`AgWo=|$$tcRBQ?@~nn zoSu2)|CEMd#IkY9PqnLYI3Uw9zldjw{NLT$-Y&=g`*LgP|M@IWNqw8r@IX)TH!#KJ z+aH(y(^0L?0_?0>^iOwBWLjO79nLFw97W0A1V?dUYhk{rb+DYRZG8JK!KK!%ceU*= zW*$HFZfB|KecoqI`maJx&t(HJ&Hul~zKY@~j|gk;ogGQ?&)VN&hCH!kI+HF0G3=btPz5&^_g`Kmi?>~)I6v%q zi=G~f9|ol}Q(?l97|6WrcSIU=)vWi_*7yHeG>Kv|AYl((Gy}{=SK&JuvqrLJ^kTC zU*7H=AGb`A`^5yo+7DCqydVStbGlIj@B)&VRG=Ck%L}cGlXk8xapX@W& z`9BFhO8)oFmtU6izwa#Nzvp@GtMPx#gx>1Zx3WcA1#TmjN67%_?mAjC)TNcg;9fcj zy!E;eL1!*X{<(^VHj45LpG|_}Cp0TG+c!-NA8TVcrK{^G!Ey8($y86piv^7t#f{}# z8uIf9+|l}|_eygs$m2&BN;5i%LV|FPVuD$YXhLLFG@>Cv*EB-~E1)?!mGjb?-zH=Ir^@>+Gl)~eDNHdxHq2eXND}EN7-N$t2keH|0 zqZMzJ$ z{!cY-7oWz|yO8*!=lz6MkmsQ&38R1?AfQTZg&~~=UE=tU4}gS7dzL#5ewu`f4uX}BJ<`cs@Q(-auW&YP^c}j$aBoh%S@&e|%d#HZLLAY4=AS>Io~`Y;MCRb4oL!2LHfzEi1Y9N z`87b(_`mL#<@>*_?$%QO^DK{{g#dBf)c4?U6eaqGPwa$sy9hrE-n{N#9PYn6Kf3%+ zfA8S!(MkVo@8aTz)ANIV|Mk)F;apXw6h1h**!$-Auzz%Nba}LQEGzGyAMRZqot~_h zlp%45Q}-37_+pQ&Vt@1$)b5YO(mLp9t(hl??FTr=UR9$0v?$=8`v*(I?OF^a&Vje2lbBb0%ss1DBeetxNRRnNX;! zcYcN&<~KTi4w&BPoKD_4S_3moXF+wlK0_j?g;PIiJ%sIDX~TO=arh06aT1WsGhs46 zG=UWx_Y^9iTm^-6PQe_|6B%gf z=u84^ym-6;Qn{B)k)Kpt;Mo0HpBBtLHb=IxN9PxtE2EeuCy&V`h6>MS!SL7az!Mrm zuv1U6Iuq%v^fW_WwLg3&wFUS; zEcCatvw>6P|4RO^?cMD!m;B#zJZ}C^s_1KOg33If(*5IcPDlJN`~4-oA&H9;s9t!z zxBM^kL5Ij`C|cJA2h-#}KMseeAo&Q4q8{X<`9;T}x4f2FK8|Mr`aeD4W9gYf|984u zW%+-5_sb>yf0oBh|38V0kN-b=|K8ljjV+4e``=%G3LMJWmVCq}Da*3WigSL7>?GP_ z$2yXo>?v2S2zG;HN6ki0qgyh^@~Qj({g(HW+`?J_8vW!;k&_v-YBCmy$HD@zURaOn zGx(-O9sg8i95>=REu|WlvVmHo!LpK#D;H_}L&`FKkwT2?m0n!0*y0~uUUAOSB=FA5 zH4@wJPo-0xG_z{gRZ2kGur#NDQi?hH1nO=3a0w|ZnGlzkzZH~fRn<|Aa2e=a!U>B* z>OmlGVdIueygtzO_1G9}NmVuhynHvCv$-!XvAOKj{sC1v*CqvxXSU%6tO-whmC()0 zg8nHAfqpIh`Snx&tR?@aG-Z=46l*$xERp}892{2ie-FOO|M&BO*FBL!>Ble-Dd~t* zbV%ny?ezsqJMufT^Lgb<{pO^}OD34rtK^EYRZ#hiisq7JrxY<$#r@r zXa?wVXAb}oGkWmkNMv&x5Hm#{3Do^v@Tj})l=c1B0zr6PGfG!Q*C%TOVkWK`Y$+_I zNVGqR58QTcY}>ie)$bGxz51S5o}$aJU#dcLm5rH~hOYeoBw{Ip2ov=xAkRihgGg0@ zbWfzD;+E?-*4^5g+*})8T}?h4sY}fLTAEi<$7&cNG`r3d7HG^W8%>$kA@fq{11#7+ zea)g#lSQMLYNRQ#MNKx1rVI@$CwL&Fl5d05oyc-R8gVY<_Y!nR8{L;QGsRV#o zsS?#tA*$hi*;R_G{wZG-%6t5*qyMGyxWa!++g8mcXjT>3xie$o+!eFljPt9`kiZcY8PM`-gRaj)ma;D9 z2Ive)XUrx%VO{tUTpj`*?ZaO%2=kfWxszH|Nfxx2vl!J@3mu zl7I^Ix-*2WeEz!h|BuWw_iv<{ zn51lVDbNs(FQ5Jl_T#sIRT}&yKU>iMl~aK&mH!XQ^#94h*Z7b7`TWb%|84S-{X4ON zte@!+s_I*V89o@Bh7gR(=0lF$>HY z_qUR?wjl~%O6XdXrf3swl~Qd)(JBrc#VyqO`3a9w_A%A&mxV7kO3{HyqA9yM0$V1stXR}jn)d=CE%F*vKVbegQ$pSBWIVnK#h~v)c9^~`qp5>lZ-!$ezayv-nxukJQrz-Ntz%I|1 zPBGJI@18f!$ME=kP9?GY(CP6w?Tu+FdgCC1e{}XqrwhsL6}v^3fsPIy!*>qImSA~W zbqORzID?Qz>}Yvm;jWaw)g>FPkjSOf#FzkEJ=>OhRThp>>FK5lt$EDJr&yP-BuujT zRPWl)?3yM2&T60eESLXW$s~hm#1ppR9Lw~-Cy&eT|NTM#YyQ7``RtH0nx-s?1W7r; zZ6-G}7LjolgisGQrrs5uGT}NqiRTz}otr zM!vH{A~pqXLw4hYO@fciM|$%=_FN*V5x4`#2oi}|Lc$Arga0D*XE7(I(S$qBXf*P8 zBzVXi$ySII#R&C%qIwcJkg!IY2I(vty8-XX_FX)iR^Kf4Y0Bidj&pgz82Mm+L#X6n zTFvvhC*TkG&!ueC+XH8MC6f`Kdl`76fU)}$mY`6fOJge(z2hXnAF!;E{1!;BvpD1nNGebTL3PQS z4YX!M%W{nylxj0qnvw!p4w`>x!odg_A`i zqT`u}(jg>cq9kHBWKMtQ3AskAgX0{lFK`0&wfrlZGO`;6SBzW_+ynP9>3&D958S@n z-z!w3qNxNWVI&ZQMotjPmC!U8hm4F9Mz16kYP^Us0B}yDjE01IpfdsZJLDZFhjMtb zdmQ=Xu{$_$2YV8KfiQ5Xcn*OV?YkG`KsM{=BRr8KIL_aBMrQOHm+kwc+nw;lgFhEM zO9*UO1_;Y{hayr{dKf;j5D#PFBK&-^Bb=-q!pH8w2Jk8N`2p@rq@qL?xY1l95ios<_loa^>3Ii1MWlN-5P5l=~+uxrVY!vzs6 zh2zpN*FK53+Ab#$c~}jPG!~1auQM`q3c}Elrvn72NTiJVvS-s$;J~Bje1@90WC; z*tM+vXs~q9CVY`m%>9>vNHW--?=yeOQgW?QKB6Vas&LVd09O{(LiPv1m7A7F_BcbuZwFV-FE7%Brfj-s z7)stR-INH`wrzAW`b0YEB4(W->6~+5=I=p_>5$KD@B&<6aLpnn#96|}thF_Ht(p>O ztEpMj`A4{<$n+yiJ45n`bfg9+fwZGb_DM%7f1M$D-0%0vXVM9xAPs2vl7)0}!8{)M zqBA5%{e99=ktO-JkL9-$8iZNG-pvvwW<2yeLx>0*5p}VpB(TB?Z^^lCy3z_aUEt>t zz*RDuHv?8Y5GDJ216&>B-`p9J!>0%P1it}C4D;2*{#zauyg^C8VG12~+63Ay0c$Lu zkTcbVU>0*JX$TQaBNB68*nn+wgUUR=1!1SMDKRG8Wuudk&%G;_xB*`kx&lPn{M*<$ zhsaopmqAn#**AbIrPF1-ZpqtgmL!3%glMh1!2l$sQ*txoAnYT@4}>j&$FZiDyCaAx z9y2NBf!sOTlZ)7l@7aqGv!zSd$Q3h6b~(^;8t%2VzXP~pV~Ah6>9VPcE?KgU(dp!8 zo}ML4u&9L~EQ3ouNHt$u3ocbzhwOSZlKZD$Q2ad`t`EO$gfT_u3MD|==p4G{8Qop= zY;HeFldP3ztpb0$f(bg8Ff8$+7Pv0kcaUOac~yu2qI` zOFB`l#n345okm!0!rnT-TSNPilq9x2_quFC%RbhedsA$o7OrjBdrftlrhQaiV77-w z3jvezP^%og8eC`#X%#z8GLiDRwwf#*{chm$meHD9OqwgtmuhQv^YpA9uDjUBVsV_B z;cn&hDx0Y#GNv5r(~z<1OLl{aej<{)x&e5NaM{IgEJYd;*YSyj@8tC5xxny8=IgvN z62x&0T`7+!aT~PDG8{S6zhxYY`S+kU)#kp zK5sP32)-T^1DN6uT@NeY<_s zaIL_5G&}{~a}CZdgeGPFvn7q!aQ)eMkA^Fz>1>INYVhTGEKZq^&-g8H^|mmU+Hk#V zlk0Q1K#=E?cCaK|XW(D3HgE;F;6Ak**(&&weZc3l0j>`nP{+IMBb~>g2W>8uIj99U=yCkhB#a~whQ(-;z(i_p0Y`?2z3P}8i{d?C3?on@= z1wQMkmG`W>-P=)c?78Gzzh{;AGv)Xy#v?{}LgqYSxMUVV`WyGI=w{kV1PWX7?)60x z;|=Za1TOXe%@)HPF0>HJMlN}C{C6p`n+uka{$ti7F$(S8YB+u#f6-Dwo0Cq`gvmKTE5lI!#5}1Kr-6F+U5j97nsnVB6ZRo(W>Ql3nznNM|(CYS~8Gvjta8 zlG&zn|6Z_aN>O>XL@L>%Pg7cGzm@!6PU7Ao_r25dUfzZ5j^b-?Bw6>!%1l4n1GwO) z^*_R9%~csK&?V~y=? z?Q>SeS1I}Js0deOAD7{4E2o0`l)lyRZpoQ-uBh-eu6vz{F_!IgI^w;z2VonkWY_}c5n2~Qaa>K#&# z!kTk0)thUtmi*t6SD?};axXb!9ES^Z9X$Kx#I3C3&hRC&@mZr6lT?!+>dsLRPdN$Z zal)@*T@#+nrPj6ulW1Fxq|2|RoL}$_y3pSOtCzs$;1$6p+mjb5xSq`TP4tqcw2iR> zngIv<#OKk&RO&2DQDJ@~#i3ac6PgkTPs(C)3(;720#}Rw#VwIdyPj;@1z?PPrhEyC zz8IT$T&w_DjUUD;kl8*Nw`4TV#A3|Xw)8Z@RqKbb9Ilj33++b(_wm48-<7Ztu3FcM zfow%tym6M(amC8XNm$`E#`zOTvA`ESQ5y85@k!+m{Y8)7#xT20+TXJ zC4lgOhW1-gFcw{}$W|0%WzltzaMa4LX%e8ZKc+&UPdeHTF!Z5@H*kVe2meL>;E;%W&dyP7c2Z;jo3|dX0 zq{Zid*^%tdm75nvErbh#`C33V2mcA#*<>i#)0b2#4(g30<+4BZeP%2T)>)`uhNLjn zWwgJCyjE|xl|_gzwTchF6<1-h-M4sZ)vO}#_GehgIg5OjEIU`X+D_CV_V)dOMj2qG>msv2GfN z-ANXK=CHT7D!oyQm!PW`>DNJY)%HyCJM+>ScN!s=SBnvM6Zf{cX_&%9aao{bHpX*aR-6+x$UrwHpBoaQ#>ZS4D^3%J!Y59M)OPdG(H8 zv8W_`zfty4+k;J|QX{EsuO09#o32aV(8#A!?%gNBL>pG|5nCtfDIG$93rWn0F4(ey zKPTq8Q4xk31XJvfRB{U%@@R^$v-&dV_FJ8MDMYXIdUtE0-ttprb*`|B zG!FMRlPB>$hAfe%c!js9DqOnr9RiiVDW<_BXnpNzhRbxWZNHW1Z34W{RPn@S{#dt< zswwB)(r zR?~rpg2YKMr^%w#17TaZ>Y`yA;DV;Moxqr*^g|x}CR_zql!Ytl!WBTy+QKjD0BMfD zaLI8P@*6=|0G3l45t<})A=QCTn6D@g_V`TZpb%U=6J3!mLbkV0c(IR26L=Qi1@{yW zbCp061nl5$O}Xv5qDz%+kPKeGJU$~dOJ^)f1Fz9dWy{yWnyRKv9QfT#ugyZk($SLaLDzSJg7*!&KS7XjF>FI6$TULExeHZ#CfAaPn2m!H6=6IR=XQrq&hoE zoUn_O$K=B+^vrCv2&PnAiJrdHtQ&JbuTI>fX-d6WcSa*WWJ%XR*=5%(N_#s#n*@>U zBvN?z>C(vW#t9QCk99AxrxCQ3X9}u`ZI3}3LE6Kp?g6N-{pcSVU9>+4)q>C%eOm>g zJ^J>RjIM&4V44Je4;U%-UJss@sM|6;#TTAhcn0plk-L7VR}EY(fwX0~mIcyo$y*9= zowdZ#mf>0!NW0~mkA`ch^JorNp1-nn^t-uoOA`j`oG`5g1YuUAHR8ZQEL%}q&kA7QWH~lA~1KUF>Zn|RMu~j z0Vp7mq2oNp#rhd!cuAG>*%ey|6!6HHr8kU4wq8XezsD1zvkTt@>CCpN^MuT^Fb(36 z0fPejIvQ=4N~#6qfnlH_ASMrjfuMU8yLSUsyBSTqnbKb52*7cu?~5L0FcUp1Io6y6 zk>Q=UMF{Jl0WqNJ3I>vd(K9E(6qODz0VycNjAx;5@=ckig`6#4Ld%sb>~&DVYu=~| z71-xChHV9MRtFL8g4zF(f+#-_5J`w9H)dWi37EfMJ+{@giL2_%SQYuQa$k2%66 zZ>3&%6A08Z%rA1YZ)SlvE6fxKl4KF6E->ksL3wwVoTd*2p`qX;<^tbnQm)`#kmJ{{ z;ZFoHh(KA6iYI=BhvPcz-O$vVyL|^0DhyxfNVInFeM%Z^@~z!Qsi2MuET*7xMV#49VI^#gT1Jypn(0 zQn)5O%$IpfUh`?-(U9QRIe*khn?jY!lFfVr6=x}2ejwzDdI{%sB=4P<2G>#~O*x_0 zJn&TzeaQ5A!&e99m!MnA9=a4R$ZzmIz^ris0F&QfHeATrttoa95b%zY5WhdMgo zu0kxwEz#$&C= zM?@v$42N9EWe4;6>cl1WDTE)>78B$(8`3{`ly>_O5HB2R;fevpvU zU1gF;#R2gkiS@ePl1&J&RkW69T2=blrfr$l>nS*87|Uu`5zsc}Yc+7!RlqFx3bft= z?FU2bwUrnE)Ie}0k9#}}p-{UG>24N5>X5qmR45$|j|S{O5W&(tCCx!50j*=H&_y_; zy>uS;O3%;jtIFRLv~KxsW~V)>-Z80Gz=;haTb)Qb;VRRg;xHl&B7MFAQQ%V=C^2}8 z7{S$0W)`Cp#waMHOtWt&fv1zXh-|DHWDzic8EZWZV1Lh2dr$EoB+TSgUU8XCi>g)r zx6}SDX#eFb0L^%W+|$+jdps6A%u-eNLqY)6L9jWbmiWj#`Fl|pm8$5Jr%HiUH*XvG zYPIN!8eQ7B`&XlL5E>L|8|8J%^vc@w#jEq5kKer7hriB_FD`z4d;W6&PBiTT+G{K+ z_iswEwbwe;xu`o^g5BMz&&H!|8!NHx7`NqiG*nGSqsOyU` zIc#@tNVb9gm*XXkc>I?#DccMCbJ&%O5d9U5%NE3qY|*-gU(m>G!x2P5`m*c>-DYXF zFx_Hph6|^d6-Kgq+Zdv460@!~+V~we`3tVb)pGacjcvkBaMi2kOSoDN&k8(EH(u)s zmH@2k<*RVCmB3t=pXHF%Yew7gvm9ibo9%KPswDRQ^*CFO(R#z&UzW4w3#*s%@1L{T z5Y?%BCC;Xse=)v>0C*}#VLfNFE@Z8~cVC<>heL9spR3_2ZJfYV-7u zaWKDl^X`mz3G-Q$1~e4z$`4aHT$y0W-&lJJJ%sCuEws7Rzy%Q011ATLs6t0a{^;E?KT55quIng#;IhYH0g zfzuc}V2T~6rprPHZd;B*`=<^rw7=KCovB3Y)!O@PaY`mIUr5iUW_ zd5F--e~g0&D#7Y^Ddj?=AI4NIA=3r_mVbVa9fe6zrgdDX+#7j}l*tZ&nhEww(ONl5 z1A|i{86^ovH)_52hEB^LNEgJZ*!*o`{kvqGNwEpl3uKM3`P2|HAG+w}tL!6YI(Z@_ zRds7lK`i1|%S#BTQ*g%p0H#1$zYFCLugXyDf@)JZQ2P>jQ3}%w60>9;2uwar1-W4% zMrqeOTomAXFW7M$A1Cvs49`;P#tOW;9YNUuZc7cYri^Z7?R)KBAURy{bRjqUJPVuR z8VU~?rXYls5ExFN-TuRGjmU5Pl;E-scpZ34;j%Bt_Hd~R$Sa6pcV zPsR&7*M-tl3YqyHcf!>lT7vXBnT?lQv6Ww@Jidb_?tvAeE*aB5;-oi$xbn7x) zW15Pd{LdZ1)sVNa0j`uz>wMO3$>@6E9&Vbqum-OBh_Obvte`Qs*nXQ>u_9b8ky$lx zwFYNxn?!=&L?NgCOWWzXE;V0MzB?phc{(m|w9%t>ZRe$&sI|k~0#{8+x<7aRxte9v zM+@GPb3@JeWGM#ke%nv$3EnL^$r34$NfNd~WbVeLahoA35xaVbngewui_Ve|^L6y( zKko#t3bCt$t1V#nmW-adPakg@tlI=vC2Y42uC{>PErWHN;Ho5Pt%IvAV0TLbZ0@)& zVBBnHdRugSG+80?hoBN%5?{Nca~3ixn47<2wbT?QC4;~l9?A3DA`f9^ebs!9&Cv-w zPWW63N|_)$W(k!)Y~$`-2Uk>noU`CE3&vN05{Q97^)d{R>fVjPNy7z|Dz9nn!|PA{ zhD_jjDoVozIm^+cUNME2P!kt$OGbnCl&|Z+)m-T8mYiw-M0@(?(aEScXK506YszW0 za0ze5>P=3!Ae+q<@Rmw#KY+U`E&BqLiHSo|U=NJiuSrIr% zm=psG?MH(xDs`)rXpzqn*2aQzFB>dyAiH%F)z9XX|GBIbduh<6oT>)QwuCabW!O$v zN-47xOiQ$Ffv7IOOd~|?8D^G3q%1}~_5a=CEG=Nxr=4j4b6Mh_UsP5IkG-(2} zJ_}6)n9K6eECq9!;M)jhZ3%(~Fmrpvx)5qGkHZ_fXe-uMcd486?cb_$oGiNTUeg3Q zS_$$^z+A@K>cMPGzu%Z~zY6WYND}@cZi;VV#Y||GlV%a)YQNRDaDxlcmUXBd%vDkl z)q}agNh>DcBkf?8{)`J5`TkHa+X{2HVe=<0QldJp2%rITnNU-Q%_QKkc{Rn?+Q8KP zhzC_(*8=n5!wt@+Dp6lsk*Zm=Sx1^FgSqTYq6N&jCgWiRn|5*dmU=DAz+9%wEdjGi ztSfgWN%0M7ys%HenqWTbKWlv%XaO^&$&{sM1z_8;sli=y`!@rV#_?IgQ||G)0Pkbp zN6&Vv8JPquG)C#Q&nP%~ENM~&Qwf?Ebw%fH^C}0~*o0wfV}dr-DolGc49C>F0vFkK zFgLjAZTUiP3yDzb&S`C?&?Tta=nA!IgtsIzN?-{{l^KEvr4U{*7OxEEGEK1t%!s8o zJh?)t)rG=Tsyz27r<;`gY&QcJSty7XL}Q*sK1mZgnFJn6XFLkgVOKxMSfwV!hIF$A z%vTD^6~N?KT7)y@OQ76Pb8X8PiGDPu9GI#BRwLr-dRHrM`Uav$TkY|CE;n8TtgZO? zJ=YsAk?@x4GU?b!tBO~|9N6VaX4ZqW_WIPo59x+peFS~3%pGsZOI6qs zyq<^`1}23!VA$hPgr#2BlZ01-zlKWH;s2wg*Fl=cGvycSmK@s;*I-oEzhc$`>>J91 zBfqwygpiaf3$LPDo=K4+#@7zbgInwS-jRYT(0EhR9yK|=YM$J~FYB2xfA*DMk6hQcJU19$M4_?!uOg-i*8 zYN;}UX0`I2y>*QyC(rY^(DT#wJ8Ny2%0nI++nbiUb*m2A8xhP~c5yV!kCX!~df zdcXvi^6p%UYqZT0NR!2_#K#0`D1p?J2oqBlW@Hkp0pK&xA6|ux8q*B`q z&|+5>bDX@$jpdl9>xQ92YOxfV9HLn6snSYiqi`KZ10zpsU8yxWn8%ZZiI}6mZE?0? zzI~P65=&j8N{(PDD8L&)bq4t9UDKo&24jt;p50DWv>vKgKT+E#Vbp&cg^MB9H@2qC zU1pZfT52lyShdVV(i1b9Fu$pB82}p(jVLG-&mWpUyV`yawdkAIvNC{h=oBo*Gd$&F zqzi+MTj6)hJn;?aLGk#+{*{L^E^H|3}pGI!F!Zi{KIYI8hh2IhKyWr7~+&Y@7 ze{ymP-?74VIzz|$#dTfluN@+JM95MG!$Bf0@P04#u!l~AG-RD2>0HQl8wZ`vzTGGb zeA$GKJ%4cU_^|k+;vJnK8JMp>!@s;tqyC-1q{bjX5!yWb+TJNYNI>*i*uO(v z&ip4ewj9O2A-ysXu0#zq&Rt}>BBh#+OYqLwWJt8pUo#?)&+Yt32decnV(FHcy zz31QLDGr9#vp@V8@~a>Rt^1EG@UFTT{yi*>KRDl#4PNCQ9w|_e=w|~vA0tL87!|Kig$JK~S=#4uK(pfgn1WP;~$-56|lz;c~ zXU^W7ij*Zbc1zo(*d_V?_~O;kVd2WZw6kU@DOXi}qABDM2T#d~#rC@N3;9YIQJnJ|Q$p z^T|NqL3|k`IF9<_`0NyO4k=$jl}1pa5r*bcj)n*zZ^q_yNWRtQY{9c+3WMNY_h|1^ zobcb7mv-f8{@fcUH1cLl^giLgx`F?>2jllX$^V1D723a3B64vEc{m)?mSh4Qhb43m zdk>Z4aFA&KU?@EdI9ZT4?=RlTLz%ERq#hcNzd86p?Y=#Wim1S=u}=)F@_W1@FtCE6 zxIcc740?W$9OPHW(We9tQXpDUCMW*p;0HN%jxTjF5mj=EZw`KtlgR(m(-_3T4b{a% zikS!XeW9oCH*KQ zh*3yRSt1C}FnX6qO2BrgG9%DDv_FVi60^Dxql~B7>X6R3U^YZj__#@eG-Z*43=wlx zf^Ot_Oo&1}kV{T5DTls%rJORNJLo; z6`QN1#T}B)pyTMluAYgKxG^NRU5AhluIv70cvqO5(Xm0 zdt&NI^29n;`|`|Y)0yI+>VPo6?*2H9_lGRKKjoBw^%uy&3#37{ZVy@E_DI=4#n8Fr zy-=9}1S9g~;9UETPKu>{MeGbvD)WNAQ?BEvWHXYTOXWpwE2dPDgWGAYg2+6Hf%1?B zb7tLOxC(H!{e57?OR|h9Rdb}WSN)!ehAvi88mZx_R59O*YY({3ckZ{wi1?i=u02=n zrwhW~>dDchdUj!Li*-~Wf3Qud&@P624`iY-^QfdVHo`QwRY#S?38cG+3rE7>D$g{5 zoLA>^0Oy_@4vAtYTS{X|T#|sM`I$AZD=8;)8i9kGJYW{7$zy-0?@QH%i*%aELzyQ| z0}6Om4mldO(uRYFzWLtTtGSz^D}r0vHuc0(#y~veIkbB;l=xyB^R$G#?R=32lCs8rseN7 zedP+qR4@q~;@Q@xg$U=|msnS2836J4&z^dI=s_mTZ7lXbvnw(R3G-UobH^$*-hkQHcJp4|X z#^n~mtzv$V7Vf=O7$P@w$W{`$dQ@PiO;^;unLdqEsk+HQKPV)iYu~U3zI@t^!R5&m zDP15^d^IeRyd7m8vhHP9rQ?#EW3ONbvag7J>)>uVb@!^~TJ4Z;@qX^>CoTWGg2O`I z$0g^gg=1gH9}^nSf|RulXkkm<(b7oz;!al>2{|s2H^Y^Z+{cAfC{fcUyBDaE^Mrh# z@Eeim*MEPySD6%t8saGVoPpHTcrqn{@2d23xj~aB(^H=eq<#hZtj-Roy2H=8xyr2I zJ4u7emW)C+wP*cj{c0~wAq<(>mosO=OlZhYOJ8{n%Kd=ewe(SW?hAbkxpGRk4F$%Y zqD^E>Mc_f%Y}F9HIL^{p&5U_t!)nXD-q*uw_0(7>_dKg&rxs$69_{p3Ge&)uXbvKO zKFNQrj5epql@dDPoY+_94gCn3O$^6!s=5Z^rER!{vF0@X4t=Zy+LUy{8jnZU3~KD8 z9L=v_1W2{&BEJImVHtc&tVZ7~Lk>CvchJdyPeTFwqhFtjU;irCe|*A{w9vrX3xJA` zf0vg~W!5dOfkj@jh(Vmi&P^OO4TYYnJUs#GEHpzpqZte5I@olCMmd@m zvO zFa=oY+F0potk{y2*)97OwXRneX$T4{BBb6G{GA2<&}=z2*&!@QXG($K$@DTX?>IYz z=U*$8Myj4~lPQv!KwRoqy`0(#7WtP6n+K6E6&3Tn3OM!_Loz^it9aU_+>P*>UCx;h zbjoZ|c?Tj>Mi*JEYPVe|7cofwVTwu_$1$NAB7tMno2g-Hjva`13S{g)G*3%j5K9_8 zBoF`lzyDjSMv$y&j-)? zHP6=_2enK=7ZJ^U+uFE8x#;qjwOQ}xi3l{|9xs;G8s6K^)?KXhysNPsh;pIky+551 zyk@hZhcbfilNrBZ*HR?cbrgm2D+q%$4Hbd)_*|9a^U1DY3>~QUT*rYOH3tD1SEgdW z4gFJB6Y`Pm1UzCN)4aH@U}-mtbNb!YzjO@};jO4!pzweWP9`i>G3U5y#ahr&QA?m!iO7A- zMsC{MW5R3U?slv4*){|{GDg8jo|VDTmxT^c|!hv{N}Za z;j@^iS{~39d!z60fdU3?dYphCs}B?KwYzAw>tQ9+T-xuJHD>uih|G}IFV-{1JFZ@r z3bjA2Q;$v{=?y0$WwC4}@Ay7g(IbY}MU7Cn-s;$BQj(t%IjW;Bdz6=cE2fIjolS8E ztFtW!=?bca-b;+RZTp8ab{n?QP;lF`vBBP3v%6~WRW})%L$1W!cw5pmO%%s-_5^8^2J&QRqo^gxy^(YYdLJw?w4AN;iSR;x_* zHMDtmeNqV|v7TD)k%UdNkS5^*pg_+NO>ahe{*k`L%F(ryTvd~I^(#2Ax=8tfo=p=v z&QBNkwJC90ax+LK5kM+OjWC*$aTbI=J_%``VbEYCS#@m(H+g3)_cqar_Tur+AE55u z@l8)D&s56je1M0s3mcakJ(Zu9;VjU!mp4+q4_5DeQ0@JA_1=$*y$iymK2OafD3{CT zUgaUpL0oUlVmF8Adw|!#8moHURm(GI*>d5LP+dr=T`Q;Nw}g2txK@&T&K!V7DA}*8 z+gL5Cx|sfyl@I;dA-}TmjPon<{`|E{wl@h>vZzv9bn@=hj9{n=6CRBj@p%RzEt$Oa z;{;4GbV0|^=?z|k^+e?SV>;9Aw%1e1^a!C2_fVKd>}^4csyP~^l-Ab=YgLO~-? zY#AhN0AOqk$i;)&f%z2_)kY;GDCr?Amy!K1AtTo=z*P%C%qaz5C>=)P7DP`;szDZo zmFYh64|pNchOrFl{qLi1HWV`A5i7(7AlKs8ERiZ0@MiiEAqt6E7K`#iewXYYP2gas6rOL9XD*YOc zJ{)N$M~I?izDr(7J_e|V<+I2qS&R?)1>|UjyAc0SxNu5WHL2H?!vA9_p}M_8LJ;ynX2v2-Wx?JB;^NIKCph(02#fPd;3Hn-Rgh8dJhIm`(%Q36Eo_Lt#}JDDwd+3GD@P4oJrgEc$Yz-wG7|8O-f#z^gfaonoh zqMrax3m6$TMyq|GSvqqeBT(R7a{1S9`SB$$^!dfO)XPAA_2BYSuJ==qrCv`>*^@ml z3*+=mpU?RK3i`DN!4{vj6Gfr?5Cry^`WqpKT6XCS)Ddgt-EMbHuOLK|SHvhZw>hm99HK&mmWhu(A##vE{|jE>fP*DI=#*%9087 z7}-6!INdW>y6*B*a&mD>{6vcT1Y#i)mL>ta&d-pZ>cjCrvxJequ?6`BOvGnuT#Hgb zqzDiSXslP;g(vi`CxR|cWe7r-%65AzZ&`6b5Mo#b&q)^Pnd^`Ws+0({!h#T$hITy> zxRR^~+&rpO1|dqgI8M_f7-uQ7lpq{Uqv%92uld5PTx&ZK_t{xC4g)d6MW3_Du%yb- ztLv_pFdhB}qO;s-ouGi6h1s-w8mS#FiJDB;_WO{qIZs(3VM`EAYxl1`|6p$%^0AIA zIDd8g^35xE?(Za7)D5C8W>d5_wUAArDg&5PURqTlS%KqxzsUWA>hinz%47r4&a@mN zj$#$${Q!RiVS<_q=}YQzf@q-+f&3|$&dBjK4MM7-vn5ZcA)gpnkR zEGd#uGLz#_dmQ}^WE)NqQx4jYs2@szbw;4ENjX7A`bQPpjhG$DnLsKVSz_79QHH2v zL8RV;W?MO_uKp?*R5+9**J6qdqv2|9=)wW5L01T!Q}pT8qI3oNloLJ%Utk)B(B$+? zsMd~lTPAgn~6TloE11 zP`PK!fBXqQ7=Dy=b2E4k=#r?3YOW9PTs}v?-|rtC9>V|k`~C9&_xt^W{}>!TeR4Dy z93DP7_>ca<;X!}!AEbXD2*^ZAlmF;%Jy&^gf056p&tDpXub(gZX}@ZK!&eHIy0OI?$zY|OgG zuk_(}FUY_>ICA^lKKYgm+{Xj?&xf^Lxfqf#h_a8?uj)w3pFPP@CC48U^=*AS>pd68 zw$5``EJVuY^&Jx|8q!4pUSEbd6SFwsAIsV{jpO{!zB_OayQ<59<1^tU0X$Dig9Z%^(bEZgHt6MZ?YtgzUk{$hg`LR)mtm z0M-ehTnaD`knSk5cuI0X1gP}KI zH<$;JFM0w7K_5no9KZ`!68Lg2cbpx_KLoEUs!q5?wRBY{aJM(s570rq8S}1Cp=BgM zDybsW?bLiq*?cHW4z)QtZ#p}KMhoLAQeLmnrZsr#x?&5V6U@$OjBvk=~$6bM1thKwt>ZqLd}fC%b-z3ivfk zLW)&;)$$VX#oXIh>%jhcZk;P6P>)GCOIZ%ajCeCHSX5wl=HLv; zhu>hXd~1iKBPEQ3LC4wAsdHe7U~9zcNoPDshh)${cygeAP)t>~BkJ&*KP&es{~?I* z5$ED6kfJ~^SybC(f)#6;+o4kWhF0pK!Ligg*3N&HHHYC{^ z%#@09iY0>T-WzQarn{BmzzXSpNtI2kzY8F^iQG=2p>tWN7b`Pz9DGbS~UWtPEz>W>FNg6ioGoha51bu!5HN7Uy!_N(O19ZsiO1nJf<>}1p?{sK1?Qp$YUe`XxYb@c@QCp zFZy|1JWXuyK&ZoGWO59e6G|SJ^YzF}Bf0cPgWQcM5^ z7e$)?V(un)h}zTVkB^?ih^G^hMG*Or8qY^778hdk&R-iJ4iud9*6N3BE!1^CeKOSI z^XD!!c0YZRgeex(W1l})GgAydeZqDUu+N|IDjDKE=RR83H4j(Flxz(Nm~qtLS(;~; zQ}%RB29*PzFp&z|i6&(98a6n>n0|ealtcvh5tvj_Zo+;+UX-|+@C359N@)S&ghXl) zDtN;`J(JxB2T$F;+jj>ztajHiuM!4AHHJdIP3r7O=uEKJ#(luZcO?-K{nsJz0%*ND z2iE~5KfQZ*cHtO4g5M1AKbfgesgdBU%H9cS*xoFi$)?&#nisYicg|uPq}1V!ask$k zm*z;0Rt#;l9DR~iFvXHBK*Lc7A(3}qszaoM*wu*fs2dR zrVtgXg?Q(h#coBtjFT&mkd7lWiI^mTG+AH~QNEK+W0_G&FF)gfBrgSm^wK> zi&`{9KaaAs)>SOsvt+c&mP^5?)+9qnGNzuM-13eq;a07rq}C`_LsW$nZb9u>gsU$4 z(Jv%0w`ViHzJ_0xZiEqyuAT{Ue0Hj)dMo$Mh~*PfyK|VqyzomK=fpr8{eGVVj8Rik z^LOz46e_bwtZG+pz0Etsx(BLc0}Vs);KBTzAcX2wTbB38K|h}hTgE5Es&@i_5TXl> z=qbT=sr*Jws$KJ(kW1XwhNW~`c|*v5_xOBHBY#LfbV{8&fC!Jr3rPpCxO?-=I-z4X z48{pf7Ts|s7Gro>P$0nb%)l`UiwP`-umzP`vTP4Fl07D?RHsJXbL9(UP7w7HE<}!h ztrk|#s;-V4VU>}B;ZLh6LQ0)x!@vv0p7e}E%1fmTrkM(-LRLVD7xK(EZgyl6BkM<~sj zQIK1-;oVwWKJY_f1p}zESURVoKt2y6OD+$Sfp8@Gp9sdP{6Q8^6HMJ`8HS}ym&h-% z4740538u5OW;z!lGBrfC261&ip!tyW9S8KcC2D&*OCT)_s_!LcptSG&m>7z!vuC01 zuF_boM7&h98uA5~N4uv{A{E3?tI75YDR>Pawd1jDag`D@o29r4>$%C+q#BUL?B83OytFW6+lymUx9Kl2L(fc#op zC5EK)>K_>mJ4I|YfT1nhg+0dCMyQ9CFw*&U;0_ME{SI+YXiRmWtoyh8wa)!9NcFRx z9B0f@T8iJweSQF}P%voO+Ngflf5cbH;*ucR|8F(#hxx)Dm}hG;d_nxcOU+O4u9`QH zzw2XK_a@nlD>H;^y}UnY#+!{afbmk!J!i=jo(M5a)B-#oUpa2H-H^rs)!OWVD|oh% zYdk`XsR^-4NQo#^-Zxb$($ChWrfPE~Ko&BPjHz&V=#; z18?!dEadZN`*~IVto42KM0-pAk?|CoYTaUJ1xKy|k$%YNls0Qur+FovcRG??eI;~i z91T$ZE+dtsqFKUBprQuy7f1+3N5v9yURBH)x?y#iy}deSQ0a(mEW+PuZmaiC@4VGd zJzjky+;4;{uD#(meuwM^D)Ow1OE_2^;_j82l#ebpNb6QdiifF0dGMXE@q{7P5r{JD zGxG5_qH9o}Z*=v51;V8J?Nra--pa=wff5A^3>ct)UJ=f&>f2QY_byt0?-Tmay0wsg zCywD-=7}B}cEQE{tZ!H#2rj8I8fe#gdsoiKKnAx$(A_>G(6P@?*i(sks=-b?}{5+hGP<`2H9;K$O4Sch0xpBnSUe+pWNoEsvml0Z*SV%`h={bWl z240!N7wGq-Tjb79)-sw70jY9iyxDr|3i5>%1zgg}k3)WPtg~2@jO(U)aMyD^P#W8* zuP|fe^p){ugH=>8&Dd1{tMhg!yaabhN5wUER_cXjJ$u zZT74%Q2nkUKUi&qHFmCmn9cUip1QhwsmTu9x)KjXXffMf)w3b?uha-?( zIPguw?A4&Yh!zX%G!*Du)CRfTs8i*gom%F*Y%%$udOa(ST(>^^))yIbo!^sN@)2s~ zGo`kjttP5*PP)_8pa;;Z>%rEYX1Qc*&bG?p87(nY-HwiLmV{nwbz za$_<_7fEs>9y?cTu?>XQlwx!(A*yN?f?M`N34 zc{_1iti1&7qbgCD3Q8qp2A*)GjwzA33j%{i3ImhfF0P(`T(^6%HZQJGC~F4`BCXO5=bQ(g3*wMKU#%l6kgtBA_d%F z(tOMykk==e2hot^b%p2jt$9$b8Z}UTp zaJsYoRD01S55cDSGDsfkFW?B$boLPP`Gx$3`9G$jr^H0-^5&U zozUFGBk42{7Z)Bje>3A8N#Pk+!a z=l?rAeEe1ayN}P8(f?>13n^PS^s*9yS_2@Hw4(8Cig13;SQN-&8z?23owM3 zYsNleoyj5TC|4oFW!FNT9sN+la;#1!hNIwFhwMcM6<{W3ntHR>n=bVTht&8~Ls}ap zKN}F+)3crD)N;gZw7Y}yN&fos`-!I>vzuqg1Uvt`(*{eu~^Fh_mUF!RZ9QW<{ zyS+A$Wgn18yOJfmsEVF?eme zx_NPwmbJFpL2B^X+P?mg+lEWG=!&Z#?p{Nijlo-Hrm-gmlS@}>6fZW&bg?mrC4Is` z6XH)VumXk%LEOjFZ~fDaA(?8i-xXUd>DIks^<6!>!lYe~c4f1|2o38leZ`_{`>G=q zRQKGX9}q?HL&E393VD!Us!;2}`W~~Ch?J(8aM7Hx6;(jkBq=BZHY9a~6ukZ0S8oQ@ zbgMdYn@2D@iP{6|H;WSJTTY%%pc(T& zsf^-5`QQhiPF|hAyZqtw^{dODkKerdTx|$Je-A$W?bY8~n<2e~5sxclTCMZnF5bRB zKY8`y!R{o8e08(|<{s&S7tMq~0&LQ$LP>6kkW{8C)rJ|PKIukeuxHI<;9X~$(ju47 zVy|_r^;B@wG?z75%0x=K|14}8PWqYr=YOhw_<;s0-V?wuHoXos`52^y8s7PkC44oH zc*Pbg4Om4@3zW1L&dMP>;W1=-Fs(5c%!Bewa$xJrcgN>HzItb$Sud{XROvx(Z>uZx zW^?Y7!^6YY_Kl!x^eMlv6FVQGcAA0=smtx=KgfY<rqS{=CjP;CjD%Ij4J zTA5_64&BM|U1HB<8mk4ebSkPquBAnkHf6IhYgCLXMmHR~PXAb06vlV08UgJNUKBMb zclNG0IH)zWtt(e|ktMf@*X0JawfpW)axc6pt`4|<612A$tId9y=f4#YqMtQY`X5qM z7gl{5{QCJf_-y$8i$rlH@86~VzmE?O56k|)gZ|h0PxtcK?ETl~VVH#swE4EE)BAA$ ztpu?zdqXV7$^Z48e`})nzm(JKo$tjT?($lJ_%1wNYtZl~aCohO>aM*7mu%;s$$9e2 zV`F94N&Cx9d?$_5p3+U0`%vDATVZZ0xD&U+hT*nxE362_|0c5WHuB%96#;D}de3k$1=krD6Kkfck@#wodl6?oBdP`_FX>ioTG*5=4ukAJ0JnbDg%{32R^qC{h z5))U;TW|eK?;X3qsROXo0j{RqeY?LQ2uq#cs?C-Fx`yXktrYrKbzCdH#&7fLjA&Wn zwPrV3-)Aj0nZCC3SgSo#>-!g!`>o(zuuS>*OA&1&X!Attb?S7**15ve?aeNBR<6y0 zhn>psSJO=Mi-f1#ZIhlcTTl-+g?xi~lI5p{s_0RMOPdii7#tb3L9g4j#U}GM&*0C08IfOVTm^0c58k zV`L-!$!m9#PoI{LXC-;8kwbsyV-P8ad8HH}asInJ5t$-(c7twJe6>+?Q59T`0kYNv z*?_n^4;#^@qX(u&>F@>VbVw;*nLIqFf^>rDnudXu4^-iHkF49n)f1{zFEuCKG-(Ng zDCJW!JZ4wtJw1UCW>Z_cagGIFVrwG$(6eM&3-c{s?6{FTKF8-S{up{pdcBsn$s z*|Uzz%EHIj&lmr!C;tImq$%sF;_h89@t0cxm&kvE{!v-}domb&&Hr>SpB=@drYYGC zSwwb2&abkVblRQ=JEUX1t92y%=ydk>NFQXx9nc;hKpo9OeGU8plAhv^ohHezSCsiW zXu`J!QA&29n#S?jsp2leY8gz3%2uc*-A(yL3U$X-^6MS)|0}^&Xd;=YHJ;AVR`-`m zk$kyN9>s8HEUf@Z&Mi#2%#-t%SRxUMH%3#JoIy@>qAnsi%Zt*! z)kSHc)0j=Qqz6?+{0%>`X7r}^MI}CfB&dorT%8ml*a}!9J$HYif^>@S{=LeJKc=6B5hpYVN$1^+DaqHA0P{H$FxiY=V{Q1#Fy#2` z)L2msq17d-&o0hX#bZ;eQjyp=;T~hYq|Tip;cWT$m)`$X?!P-@;XIf|JYgHnvF!do zd0fu_*?;`>;OqUrkIxP{qiM<#tZavyj7EiV7KBihHKyJboigD%s&cQuB71^}84E)q z?>E|Yf@r!=5*E@lxMtuaZGBH8-`OD%n_@}b-58Q6F&}-S|Hqz-b)|U(JtasaW>A=q zxsLnt;_@Qp33GPH36xDCznokUKS+e*PJ^@u|3&Z}cl^(!2mjSy%%(m0AN{Aej(Yh( zW9mtY2M*cJBUjwS&LekBubfA2I+y?ENicOD{U2wC{6dp}XM&u*d?g$=PWbQ4OC2}x z8SP>7g#Yfi*TUmI>wWRdy~1ae`+xrG_~n~d?%dySj`sV1a5Q*)P`v+xr%w*P^8b7J z>{N<-JI=AL399)U6*vnlRRz&bqNAcs)(| zdgY<^z+9IwnzAZvuv(8sCK_bMf&?7&9tskvlHhWY_%01~Nz+*c4=y<-A8ORP+GU{j z+>1@~t}X()S9(7oC+8#(goc7^yJudU+*ly(#2`{g(#A)KNo3TU$z93ojSEvO+f=}I z9p}B&R0)kx`HVbiHuxb|8=0j3z#dn6D4nUOPN+ zG+}IjMk?(R6!tOZaE)>x1_w{wzT0;P6)3KKNhWmgt^iP&FIgq? zQed!Tq+^;!E1W!=fBn*3$7=Q!`IgDWqhqB{vwciPWM9yLU7BCoAF zIv=cyrb@u4i(V^LZ0)f}lq}&o&dv^bpI5}^HvX>%?!lA)={OfDO;Y)D5LJHOCm+tZ zPqa!y&Qy3o4pcF-jL$vM^LXU3I2AqZ@70UB-_?I~mEyIBO4FFJh$L*zubGcWM54rk zKg{P@?yU33DU&NhVl=7BcoR zW?qV?CT7WmCv&7hU8t#~ASlx!Ai<+XRdodBL3u*KTsM*`Yao?Kf#`@6b}c2taA6vz zNybL5^Z)+e|Bv$>dGzRKp0eSiN6vTTw4gky+OtwVUYKDpy$oKm8!8ADf_nj+Oym{O zLMo!UuTRa+1QD6gYl%+DV*#3PxcFiJ*Q8~e!Mi~vm1@S^gBPlSLMefOS6r@=TxH?w%_# zWpJg$CqbqxB_hqdD`PhR$o5H$Fnc@_P>p=X47HKh5~D|NfVyd9z_^7KtAtUK-^M(W z#}7zD+gVOITAlY1E9wG-p*lM?#|tw=B1sHVdFW>}oaDD(%U`G<1|am9_d^X5@_1u> zHYCE-I)fJSoRo>!v9GwsHdKu2L3(>yMi?_mIDBK|Y1|13*jyfGpQSVi3tMAeZSy@h zn`54xP2u=F6gXlxq<)ns0>^QZk*nq66Nps;OCrJ!>Bf>T zLP;`~N|LEKT3NzX>yUt;{Hiw-tBnHyRsr@M`ImmJaBj(&o`BqvmrQs`00HOZmc05% z=W)o$t#eDd@9C&#cJx%DVm zCOgW-K$O|h_f!-lUP+T~C_ct3Ap}L4kt#|3t;!|#-SAs$rtfk|MnZJ(`04WzE~!%U zTmxl(085NweE9{vC8JJfq)`1K*7Ky>ZCLy*ZWS zqW6+}83YzdhD{p5L#EX5KQ%)okKM8>?Z38}Z>2ddDbKeEzMtY~xv^ zZ-?^oa9q!ervAT3K>-lH+F1wP4j(EMOo7fHn-s+HOIr+9Yl1cefs zoao-nSi-EEh?{#@X$W@!;MO%He&4~GWdSwR^ni3W* z3hn4xr(WzT=*vWPAy(B#OgO}RLW{yR^YPVd6X6_ zdR247ffPD<955->hYMvf^Oz7cS*YxJki=5z;SqVIQr_ud zK|35TY~j^41}Y$2Fe)^YdR~JRsY<5mNy(X@%6pwQEx&oNTx;F1bLp)Mt|* zk}`YVxK64`81?b394>)=+fZ2|ji7C*?OPy1wr?T?@@A{q+**v4iXBK~{W@dt;9`*_ zZ3S7`2GBsJ>S_Q3wK89oF{z+0*~t`LCS&m1*ZZeP2+%V{;*14K4MXcE(yk48P*>eE zJMfWb_;`w*aN0{YFlL)lB7eCzYZM=$3BzE9TQd6aTM2p9x?#2z$pRl}w`GUgo#GpS zb);nLEb>_rf|bTL9G!8$WVlCMfY3Ro5uGC0wx`%@+VGmPWyY4C&BqL$KEXaKA+lhe zQL#?~bMnAP5gCl#tzz{mjJQWO6Qz zxWZQM29sQj+{=+;;gpvrO3m{F^YMDwhv2FvT|)7V=7P}t)+%Nux1nFoMB)1x2n2HIkj7x}Rm;L$ zgnRo6e(Y~v+~PP4&;Y^{pCzt3B#!gwT@Vl9B1?fyinT~LtAgvdQnb3gVvAc8hrdI4 z*aP<^48gt$Lf@l_4?9S`lqKSkOC0BdvE1V|cjmqWpVxe_J=_X>zz5F(aDm9rlO-3q ze+B3Fub(#m|7^^T{rQDAV}2I0q;~>dJ1gdVv_H$@f1W-q`Trjbo(#VF|KH08QjMr= zcMc&QO=-xd+G{M}y%~_#A?ZLek=iEUZgZ^@dp!K^1sS*pM{d8{C*P8R`*N_gr-BWm@f7rnJscqX%JN4AI zrnYUno!YiFwQbwR)OI^{J9V@9zwcgqwuie1yE)BKu3Sl;{O-^5y)oQyTs%}o$YkJT zEeTKk|7{2Qtf}cC6AF6t;2j{)5A(c)YK9L&02eJ`;2IB5;zp*xM(~GFh#-4>b#g&p zUS!Tx?EXCF;s3!e#YeB$cZY`Er&-N+P_q}05VZIMIggn5f8`g8zHxg`|H0IAzu4CR z!qzV6uQK zpK=3ZftE)VwVAKD6+QV~u=i*OOP11cRw37R`yxKRS1v}$e>#tth=>R|E>Dnzkk8+5 z{<#~jBt_tQ$>L{sQ5nWR+xe-}4KMz)USd9{oG1_K&O>_q>@a#C8-Q^xzj8&QSEsc| zf-@arZWSO+bn!f;$vypzhxfVe7Z>mBEY6}(PA;P>YQVeS6Kcl2z8hM`okboii>teo zM(vM@LRcKcJkXs=*F`Q~_RB>grYI-7LT15f?@XI-CRXe}@{ZI?9B_DSLwj8Qau ztfuj)J|;f@_jIbH+c1)ztOR^#rG&s^@9k{5WQgcuhWdx9@N@o}g^Z(iN$jUAwGun&u>n3DlJr&@|giemC z`vi{dv~jlCQAYdqzF@hKSx{ZTlhREk4Mn-gN~&}Ajyb6&HTksaGGTy+QL2*$7z#~n zjhD?nKYXA2-odf`0M5K8T3BML@sXf;#ODoqym4%g_cyDlbq8@6n4QlXLUtEaoBBvS z{a`6Nn9$Hoa!sRDWQ3YaOo(Vr8x@FYJfNl}Prz;-=rsQ3?=sp|WvND}?h4Zfn-xtJ zvJxSlyi$>%17i<~Wmt9Gl@$?z#{8naj%|8T?h|Fes&nKf!4Sv&ii?9EXr#k0m>&FT zkX_&I$=)L7#}E5Vl^Txy38dwNjv{z&EiKQM3 z^^@~#g|(MyWz)q|+@5A@8+wAN`P0#646lVQLC6|;M)6?1_^cHwO9L*c3EjMCf>@6A zbMytc^_FE?SPAMujFcdnv|;Sy)S=mep9n1u+sXkKj6$>}RbT7yFXZuDaF%ov$zjWp z3P0zkjF$n@c`Dq_C1?8I_Ofl?7$sPv6mhtj$2ZIMzpY$s!%x0Sj?UMNeZz(<;f@}$ zAL!ZJ7FM5j8&GSPbM4QF8i5xxdosrj!)u2=j1t0e>bdpevI0-qXE*tXjS-K^_R6Sw z(h=YLsic@@fvYw68y!U=BMW^tI!7hMzM{`9Ck5;e2`TtQqS#03VZywRv$h-xw@U^4!1*XDqWDbthsOj3JFo+pwm%`W-#>SN(C$HWNrbK zy~)s|;npqL*{+!p-u*z0n>!eqQQ90giJ=4b9A9k5s5r&<$nVhw89wQ3oPCgULEpf( z*nUaU0hDtimXdA|S9v7U#`Z-T<7Xng*`}qa7*L{PSYrJvusN;N5C=~gxqfdz+%<_f zWZVjA1Wj?Vp-&oeCwCC;(Zfn*m^Mk}!Oiv{ctd>mMDRum&NoYZ7I^SGdJeCLLlC}l zRu@riwRB1oZ%-g;BJh@v?x{n43gcls^AdI?Dkifie`(B)3U|tQ7{>m)9I+U!qV8OZ z)FFfk{i}iliaE{uHYSS$Nyt7i%)`Vkm6Vl;jVAcEeKJ38VNuq})Ng9Yl$7x99YuxJ z3EQ+dWFgFmX$k9N%9Lqc4=tXi#EhpG{>r=we|Y3I-~?jPg>Zg1t^A6$1ikVmmffJd zgO*i^P$vydWc$~a87v$QiJN4&3d)DWB^(Q$1Eh60FQY;gt z{dR@*2oNu*j0~$5kX?~aYkv)M+vY_geVES%y0?xS9I{W^<975fH~X?|>q+`m0b+8@ zV3dkuOZQNY(@`OR7sNE4-cqNtPuLHynt!X1I!rdzB|s@TPmWX}H+_gvauKs=n;Kw8 z+ftWX4934N*UdI!)m?#@0t!7I8_X|6gdGR8->`bgy{<0Ovyfh&*ENwkRjpZriLUOj z|HJKxrCDb`z3+I}1L<;4{x)^5fpXjvAU9KqPJFzJ^i+SHww>!VN&poCFTmDx*Dcj= zTz4@D-`qLEV$x2p;-^uHqhd=sbRA~qhHuhUyRzs@*y*Ph?~YtCSWN!QmWp=FfVij1 zFck;b>S1($tT;`vn@|I%%N}%|{L#h*o!1_0IZJ4mzY6G_)tSahDM+i59`*3`PG>W>5Z8* zLn=bRs9|q?G1x>@mX?To0BcENjaZ+pBiWPjP!1UZXW6-{ZMtv~MJo$#-{y|`b^exD zQNE-8ub~fzjJevOfgg!BjB;vptQ!>^Rw>}Y&?RhNp~DJyjXR3!dG ze4*Yw3-aJgaHKIwt{11Xlcw=PYE{s|U~k(|j+2%7dCcuTmY=_*&M*#_^?OY96HaZG zBW-LnjbH_=^x5a__s{i=n#A4K!u!=*2>;TLTQ8sVu+>^0^IWZlw{TggNIeGVsdC+m z7SxqwmBCOU+HvVBJ3hv>>%wsrv6px9CH&6o3*{}2l^pzaXp_U2&GNXH#Tviee0dI^ zShD|UccbZna*39em-|l%U?;z~X*X3v!SvUxdU29#O#HQp)#s+C2=7+ziaIshNl*~o{gKdJ; z)R#Fw@}})(Ew$cI(*V;xbvt#G@!P&{5PIbjRXuji<*7k8-r(=R zRK?bh&Z?ohwDqu+*Tln3Mw~GxMQMT1laDooXYHv6<>W{IR7Z>rn*$w))YSOYMFIw-3H0PSkhU713~k!< zm^yw~{&}=0#;W*rPJrE@z@*cj1wGzYKzs!kqPOS&{ zu8kf@Z<18)6Mwu*g>{}3N?@>emX1&n2r*xUU*Q#{=Iy=pKMUAs@?SOzB}m8u_&k`? z`zMzGsY*zJ42%({BvC}++R955>7mR zs+H5|-U^bw=F&C@>iejRd@43W)8@DJvWg_gnkCS;%QdlZA~AoF++5iSIa{w(vQ;5Y z^$D7p6rZ=1d&|=f>o@jo24GjQ#8_c34zxJQ@!=}W*vX>Puj-_6IhYz} zdkpOrTT(jKswJrI1+k%;azuQ6^oLiSM8?ctO3;w1L>ON^9A*_NV`q1|{}^C4J!X6k zT9ScdS-=q`T)BK*Iq~e)S_oTy_|07^2w0Y1)nkYRE0y!#30+hWtlQQ~WZBMU|gkr30s6+`e}QuL$7@XhXc_ zqC;5h5q=8EuU=^bD%gmxi|=vVIU|O|>!Uad1e-q_aERAS$!#86KSWAfP7L~Gf8u}A z_9o3$2Rl1MmrNNwH~+XNDoIp|S1oHBeU@TH8Xg(Z$;Hf?kc<-S_4hZAjHw@FY@k72 zWKCUqStJKKhyityFf6sc^7Z>PmVlrQsRI6sK;nZ3t~Y~()0nA0j6GQs5wD<%E`Hse ze02JYQm-v@{2=sqrRY`v2XI8P3*Ut~FZGmWTvUw^_zE3&#^ua95`ab=RU&JJ*q{9; zauO`A@%bso`+v2v{KI5a0wW2d5_faCXs^n8ZQxq0A@f1ZsoGku72Vg?D?ik}PZe`C z2ZpKqN;awJ2iEnk0> zPisai%24wh-fnnoaOiwb;SPEq;itQvT2`UW;8@V~*K%f(Al;bHGY6PjNRT?Doyy(9 zYzoACcI{$pePYTSl+?9 z>;zmAK@&Tu9jP^=HUaVX^xx5l&YuH zDa}vVR>H-bCvP`CJE^j8v%r&23_;rJj)$b1`#60}9OwweJox8jkysBd`%b^!g+IOz zo7Z!S{4JI%H;h}C@)V8AiB)70AzA`3gpIK>s4K(JQ{s`@?`Eu_l54}JLcj0EYJ}m~ zb4-VO3WU)V4qR3w`my-u80S@0Hd#X?#^OzRy8W6nf?c+COl zZ_2h1C#FNE79)}12+&q2j~Fk^?n^lpouhlL>6Y4-Jm|QM6IN;lI)=#S_{q}!VuSJg zFUo)W;iT9%{9uS?OD?}BPTMJeT_$k&w#3s;H8K*^M`l4OLaTETh`A>nC!7nX>jvvG z&t0(H1;&2gCCLZAKK4jA8QGsNpMi8uHUlEf5`Y{A1_oiXEvU9~gY1Z-83J^A!n^gv z^z?74z}{ge?xrB{ZuFXbHa^|BXMJJOhVDe;+z6|o@^qHoeL42DB`PxKk~7Nw(zs@| zS=0UpFc8HwF`ORJ(ru4jX&Zwxadiq>JJ;-gXbp8k8EdRoV6~Ds#&}C}#760som_O( z!tfbUcX# zz;j^%j|X6u2d+68gq`0))lxVt(2md8;6${#HyL;c1Ju^P6I7qS9DP2h3h(A5-4PqE zff%JCVF z!4@QI$Q$pVkNQgfN7hJ6KE3&W*XNt?M|tqVeEk^)%LDGYu|%NJr4@QHY&hW~xhV_h0wq zo6jS`fgrI{PGQH)k5$3SJ|LI3pX&=Hkbn5p??$lF>MUzc2;b_W`kZ$G{*K=u+yF9^{Vao5Ko$z(fMq*Nfwn3rc)WdzWuC6L5w;h zcfH>xs*c*4DJ!UoRY+StfiYt9r3uZ7$Gg-z1z8_$m8!+pEXQ+)`C#G9kt00>V)CxJ2h-k@tXYu zYNoZ;m6{mN$!(VXxsQrhN~F+cZlrvjF&~cD9bv0c7Z$FD!fiEC`s&NFCsF5zTh0>>cIe+2HQi|M- zy8caO8;?{NhNHCTq`jJJq-AvNEH0{6ysA|DbV&hZ1?D{hw=4C4$cnV1O=-+AOh;9;tsD<$iRA7@hdr=n0*U9OKOr*DS36p3SU`>jE z9%Eyji(2sf+)|WfAXoDi2YfqN63@h1UMHtf28;v%YJk$>T@3Pyl+>e{bUaX9fX<+meD?48DK3fTv|+b^bh0Wec4y00H28 z_~+^S)59*WXD5*4sA+}v`ig6;k-EHl7d&j`<9LGgMnj<7swMwub2+%1f znw3`O%O!`?Z-;_pwlP8~H7qCc6birG+B}mMLF>o#V!{d4<)mkweV%yU#}G!8225(H ztc;5Cyp_8=1tnS>Iz-%I(~sUmLN;_q4(qfOA5#wvTInWn+0E@x^5TGhO2X@6nQaI7 zeUftJ7#))4G1!)5MfooHX;K{Aj1pQb;>1V7@>F2Lxj0}5#I2Q;qRc4H@K7$>302iB z8svC|6yMJ{(h-w8tTnOOcph_suXRB8UYf41o5mz$caPkgiR`F*TI)IISI^q8fk#%) zaeHtH@X~~j)v-5CcWQ+R+V+#PBM*QDEF4w45TReaWlyh4xVPBJx}1?i z-VA(+>SnpB^~g8m-Tz?C55x{?x^Vm?B5!$>I3nBllouD2QT;3DlszrOpUyvTLvQfb6dz z9oyHpgYyYV`@MU|U5Z&7@-jT0krtBLbStxt^sSLUiDKF)jkR1t;CIyKtx04ZNY->o zgJexAR&l!8$maRjQ~U0#_e{#@M_-t9zyS z1=1L6C@G>|w2`F`=QkxDBli^M4{g0ctjj%+tfBv3St9}VkMS%0QJt&pNKQe>e!^yQ zkUR_qBx`c~kW0NpCP1==?~CjJsQ>icl35jfEMW`1y;ZG7gRV(mS;{eBkui3>1~p$O z>CgmM!JZpFZ9<_1H-eRSG_Nsgc*<*0K48|9ETn*z4U?*(52Itam&Dg%`mra?luPV= zF20xOAPJvij!N+YrhYgg`X|Y}4y)_-Z=t8T!i#wk)jn*UR{YAV25nK?a1uX1^&bWC zuHTdYHu55Ib)EIj4&w0v4p2RU`TmtHsB@Ihm-YcvH4wRV?dwoA|3F<9K) zL=(Q%bt#x6IHN!pr`jhmwEtv{Dq8t}vgW6m1ZvunW9CsM=bZupgHb5P-Tsf5IJEMY z`-ViXxS)(NoY)E2Ku^E#?KWus%wIuoP_YIFWrY=_&7zM2*zS8Xr z{C6CwRSH&HxElPRdl^}DN-ZAfP-J^f)arlj%~;Azq@&Xbl8Mm2DUy=gLs5aoO(|%5 zVv6+mA}_jnOU}-+VST^QxNN>zT&} zQ{x+olfov&Dig%At4)cv#h6Q%vt%Q@P)Vx!9YX?|FJ08>dr&*KYAdopplTt#qG}T6 zy?Pm46mu-4M2<$bJMutVoyqKjDLftZIuo-@1AO}>&a6S5>MYnR83f*o@V;h>MB31! zIVG07-`{`uG%~m}+3vWc@mY8hTR9tV#z&F(>qtUDx%h0exOtavAqiFLhvM45if1Au zoV|@;jFT|b8R4TBS`>QL)aHMf%!J~qk+AHe#!4w-)mnw<*L4Y5(6RDjzNOSRH6UH- zOmo5gY?NZY3ktJ*%#uYz`f^bWZTWV0`1^E+f%cSpWp8r;dIEJZ-MlDE zoL{0S`t0`$B`(#l-tYuAoDGtUt|q+)m1o^q&NzF!(9}kdfWeK-VRXk*mf{u-DAyR!*i(p!)mbt;GQ1)ml4-RPu$&H^1;o>(UnF_&qcw-#`q7y5&3xI*x-%@509oG(k$?(ww&S2K_PsGH z0{1@n1oCxDf*AUWbBdtyrr1_v))&3K#w?*^2p!pq$S02bfB`vWGPRqHk_t3h0)?2L zOp1)LXS50()3pSf#aThnXm6AIs8YLO_0h%+hG*5~Z*irZbsFr}Jj(4H~V( z{nJ8D1bd(~^r2KwfnkaA9LU(lSFgK(Wgx7iR+oofvN4DQe5Xz?nv4~e5HaeJO+cVW zl2WwJ65oR!VQBt^w2A1XGQH^Yy~HR|HBZ#&r*f@W((saIh@Pq#Pw+7NdlQn~2}stw z_Ki5d#yd8nZAJYjYhv$W{*yKJ*&Uf6S(99p5nh&bRPLwGTn>^oVNmifS~KHypj)&y zALw`J@dS%k8n7fK1;yP#J>k8eq*DbJ63`Oxb)WUB0Oz(Iv2Ry8>vEGzt%=^GrX3u~ zE_TCMt2Oqs0A^-r0d-LN=`4-G z8UlYSM;`7mxKv7=@Ee$dg?R#O_k(j`mc)9{Ed*Z47!q(p1dM}i>+m0wC~4n^YQ!pz zcJ`}jxwttf0Bv|3qhyJ8%FqkogVEC$3@hm1eO$F?x*Zs$!m~Hr0t{ks2aY~iMLg9Q zwD4(WOau_7!v_>u^mD6#c-Nd5*Cx@Qj>NR-fkC|mA*GI5%ATPlGeV?$qBO$D&;|{F zY4Npnj9U}`W2dPHN(l%lb+P7Ks6?*W>vt}ae&G{v0vww}2TrKfn|4b#&{2IGEwX?7 zy(8?t>OwL&2?E&192DZpyL{nr5(PRd1u2GMj+V1@?2CQKVR(F-c~H!{?$Jj9N1 zAO&Qf;4kc9AdZGBuo(oNjzkqF5O3Fv$_&CH+S9jZ1R(D; zfI7j1a!d@JZd0V+zh+CX>SV&p&Cq`R{?6aG%32s>_|#pV;WGlj>MgGhNFZ|o5XBNH zuu3dVx|P9N2nqOxbUgk+j+ml~`Ru*4olw@V7fZy`g4RNo>T3fU8+?#5sjbJs9%%S; z&L)5TK6kNc)iMoHqiIHiLL9??`AaW6dyuMvCqX3HLX4{MsS^t|N)cR>)iA@{>1$MC zC!zr5@-8{o$MLE~k%uXPR?twYv^mW}&3~o9(T>GN4xbXOyQo8e=v*`0KDdTs*pCnB zz~#iahR68Y)OAkgCt`qgmNK1{al=nC@{4lh++344cN8bJe(kLPCe+ z?WPxYG^S&Q&Vtb2E+q!-R-`V&wpvSMbuk?gl7D>`Y+46(e~&BTij4J(&Csxltb1)0 zyR{jW`HXH|rQL$hOy{e^2bBCjY$%4iwvHuCN3B*sHyp^~fm4A~Ej9Y+?n3N4zt zN`d!AM z0L(ErhKN!T$1rnH<=}9a!EP)2UvkkwR0|40CQ(GbFpQD?-Y(WhdW}A${iV2*{~RxN zP+so9jT2K%jR!sge|$s`Bkhbh`jiThJgjQK!2?&@Bnc8|lKgI%n8qrTf^N0&hb@+oaPNQgv$M=bPkoUQ>H=}(o4nQ02IuqTE)NO}yxrn0PFb^< z2d#{17q8YW4iG;SYD$ug_GF%sD5Fk{jc0-nRA#hYti%nO({U7OfjO`U-C^Djtc6T> zGN()#aHe+Q?pH%iV4g5f&p!m6@koizD&Cp>z(P4>K8<8tt3WHkDXTt`BXAC+MSN4X zfj0y^?cs6Q;0J7xBi)Q{&1~9stIr;EIrj zbW0gt(kFBqyU2-u%1pWXVr|Trb3{ZxfX%=ZpK=xSF)D4E$xx~85?J*rx+s|zarC5 z`_1KJef06%Je0x7J8LudO|LpZeZK<|6JD=A^j(8`cY^;f0T@7q2qwFWD6ekkQ#se2a1@%7Zg#fr6zL3J5#ci_W#!ejIj_qOC5Nkt7&`7%*H~h z(9TqnIz7&}*a2T7X^12&j;4^TZ)P$SI+1?sdmCN&=ilBF=4nPNdb}sP zN#|C}F;@T752E@mc3T!~4=!~Lnnz#MqXgDibFZl-td!6ASm>@yx^2{<*f`CWCY*tF zl{+4V3mM3A{6vw#-EeF#-a8|5>Ogmtuu^pMA{FKLNAQp$h!1%mUFGuPG&vjKFeC1= zT3x~-mmNCa+-bNb{zJje&wEv*xslc1wwb0Qsk=6%S(0kjYUF`e^b{HB%AS&^Qos^1 zN!UHuPO~loBZxO;NfGqt{-LF?h}OVSsh_5n8b+Rb20u@PwN?JfcFBbw1AjAH%1~nB zoZ)Z=vZ~UsvLX8$=cyaSEo9d)mY4N*s`qcLpTt*0-=;qw=^_e)SpS_TZ)#a(DC69^ zMKbsSF(s?Oj7&J@Wah z5KVVEx>3{#jAC=ENKfp2EEKl4__cDP6&{@J@kWEN^4nw0nbsQ~TNE_<)cH0calfAz zn)h$L9n*y?r)S+A$I-%0tfPq;Y`D8y0C3Z6_x+Mo;1~NNd9i9nChr{~8zCTfIj580 zC)MhwLW~?2$j4_S>>sXOYtnBTN6v(gz10cVTLm7Lclq=4v1YY(p7k-bf|5Bqv#jyk z%|MA6`z~E)hxzLB)xRm4R-F;}mq^>rd37C(Gdb2aauy*hbc^C?TTznFQL{G?MvTAl zoy;7UOQBN_8Bc}PT*sdrj1m6+-4)msd?Kz^2hRtpEH64lEygRFdN6_j3^oYBI9r_$ z%~Rd<84&iAzXSKA&ys;RU;;w%lbj9x9tYp+|3xi+ArLzR9E4;jg6p8_F znoUU)#!MOLCxguAt+TzSfUNm3U!te=GX?D=t=MVu=MV0^PqO_*!-lCQ^~FCF!o
  • Vj;&ySk_p9C-DS%liPh$IQwZ47&w757=0yx$l9&I9%jjXlE?P#0 zo+6*1YkFmsuudMQKeYMg)=oEF;77mj#aW&E*}4#~v35xF;@^YHGHV(2r(VybWB$=g zvZF5s5v&PO(nu%F8YN>;cN}DkG;@Ht(*#YJ+m&|vap`L`W9`R zM%-k8%P3ZG0Qe2j_-hsvpgG-q0>H-Z-gs?Kz#z43W4SP$>-r>gmBBp3jLP2lp@R$z;h+Jl|2`_K zvFDhrWqT0x!kiVW;2f*xWDuEw_by9#6Yey{=F6a!$vfS@L5fXP)7YIfgYo7+9|)f3 z(nTaA8uyuQiVCv@uH1j~{z>!}v9n$zUOfJJA(k!;Y;X6hN)};zmAV^yzL^6gqN%r2 z{Tmk6E_KE?&^`kCoclAJKx(G}%T{kz9{YknzZR0eo>BA17Jxx~7P_tW=PPm&(=h$A zi=MF#+TFwP_h!Lfn4F%G+TCB-IEJuFr!juBO@+lH0qx|fQrnF-zh`|v4Lh6xktxzt zjmUL4ujT#1VftaoZ610TWXN=PbYbkAQU2F4p}mvzwuWWMP$#Z!SThW$gRjMt=t0$P+DA9E z3!(yw(TU<|TlEMILoopH)00UWqAfMm40mekdN^riBO-4HPi)gD%Y&uo5Zx<2Eyv}f zL~Gb|As8<{?M+NKO~jy-?9OZIzWMIG#Qq9q@%_w;@O@>qv{Q#RL8|i;F7BOYp?Uf$ ztXS8<9rk6zUnPuC&xRRU^PIj$nclhP|)!PJXx&uS09B%+umsMDnvnwZ}O!u=o|5aa^ zZcCf^QzUe$(A5s&PrbEPOXfu}@>h(~_e4P^?Mo5f2&jC7&CT2`YTd@rvD$;Jt}spy zXot+q(HvYkCgN3E{cWGa}4+_NPq8@60VXPR66jgIJ`N2^HS}3XRRh`LY<_4Qsv9 zOLS}MbVn~F>yH!s>Q<``tweeF0SN3j`7gpoGr8%$=^l@T?aV?U_p0q>*w)=cPD@V@ zv8^ZByYBvB7Zu`Feh&Nk4)}?wQMyPhd;L z?U7^(_9-c@p^%~+(aTIIHW9YUUv<^z`uz%Vnr!MV0ev=QfMgw ziEfC|21nIF72|>zUxkE$FE%VI-Dfgg^j}_P%85kt zDf0D5(|{rf+2DBtXGoi!hafL7Ms=Asa#dSFCae2R^i5-T zeZSVDqRjx>&F8`07X~TjhdYjwUpMCEOkA42EK8pZN>zj!zlVB5kv(6Xn%t3Yo06*z zj!jSLh;!E~iZ5w2pQ@2i6+~I&MP7Ymw?60m1;EDbOd;+^B!8;aG`wjnJ!qI%BEHOd zY;U5i8bd8{!__SRUs^_@FHKx%GeA-)2sEoP_TRO()X}K@cPpb$slQ6FOsF$uuBqW% z=ZU2OquU8Nb2VdBIpzec2|zvngL&>RrKYF3!twI>iu1BH1B~W#YB}32x$B!lK}Np# z+8x4y`T4aQ{@+tJJOA635wBAqA^&K*i};m^((tD66;JYE<23@*6s{xKjjkiU-w5!; zK&!)M|3C)UXH>M~(RQ40|H=!ZAt51P2v^85*r9S)hPr~=HuXY0mKcO@sW7B43S}JH zrm3zcLj^{Uk3J^=)oWqf<-T}%Dp!F%evcq9GJUFaB#8I58gaHsIJF~Gyn~)6uMPgh zJd38vJJwbgf#?-SZs}g?5@)O=j$$c&#b5(zHE+7T~Xib=dYryu-aF|;>6NfdSvU3hoR~M zJh2z|OJ51<6y3mWfb`f4)|!Is=(zevuAh!TUgFULdH5H97i6My0Yk~R=;)nG zG1H*+Or-A87wf{4Lb`T(h~gE<^`Q1xL4w56B8y6i^F1slJ>@Bl(a2LI3i}*=7-`!h z2(0$21UH`-XOYB<5s~xCi69Hq7GIf*xb;M@pFj8Dnks9ydB&<73fkqHt}`JD#tp?Z0> zpM>2rklNysL*4j8Q#Y8LfZlQ*=&l6tiJJ!zJg(>ID9so2dz z^nQfsT!`U>K9R{J^WFrILC9q~r)5y>vQFh70sUxi5YusNFP(x3cpL(`r;dpn{l9j3 z|8l$l7rv5#r@{gT!N9u1)m>JsdUd~RATINP`sden*ci#u++aizsY1lNpGfFYl}vy* zo*DBxW&7yHZLHDw-8zApxLS;QTDd?M7Y1k*W{u2%@dG&mT`ZS}dsU3fKp%+@P=gEj z6+*q_nPU#s1NxZE0;5l#KJt4&A^xRT|A;wkZiLAsY|V9K54>3T^d_#r%4#oEeJGPj zUrE(#GfmTYG|u1BA}_BJT?#pI&r-t41u=DWkkSOZFjhyli79an#n%eR(kYf94GX}o zLleDoKxiHzWs(Z*wB)Zb|6h0Kh12jCwuufB~Zw=lN2Z26*hDolR7D08={0$+jyqFiMw!V4oiekMfgNpGY-}iB4WxxNMyxHbIO_29YanZs#S=Wm>9# zrCdw-I&oX?H{}dAqn!s%R08;hB$OG3*-n(r2QkZUg(Koj7VUyi&}F%mCi%)yGwAFW z(<=dnBNYr0t2ssrddnL%~jeo$U!wspU82kl!~5Mj7a3)NpV?J&Qw(+uTy^aBhE6K!<$ z*+B-g|MD_Yam?XblLh#%A@ue`@l?IUJz&?=iC~B7>Hu_!yLZkxWd*-}eC!41%-<_s zC=KMuLP@Y24GB2r5n^e$={aXK!mNvB29uz?%qZz2E$}xa+>NK1uR1_v1(cV`QMi!% z%Z1)H`m0@&6z1H$IL1+$`!FIUvCNHz%TPsO7L75OtHL(+e&d2)BfDs8w$icN4?bs+ zz}SE)s^2Sbx(bPDk~B7m+}Kk-Cn7V@{7!x8zq}0Me|edS(GUHQB-7?2fXbGo8=OXU zFhtOO)s|c}C-OI3sI|5igGHVFtt2*Xunwv^I)rFk;+$k1=Qr+*6F922S;o7U@CcyK zN?*Z%qBNnXv+Ul6dW(L^w6`?b0GLc0J9(RkFSvAEyb)K&@qc-l6=TxLzqme=^yuGD zrGJ3&9EPF>6;3Or6n`lkp}+gx=CUk3YbWiaUI_}mxkt!aFY~1fRgm=7` zFQK!tCzAcusHn$s7m&x zV$iRc5_>l1O4Q^51_hG!@ZL*HCC7bB=$L{6E3tt?hHgDlz4-BZs%C50Nd`vg1k-a| zOrG|2q56%(MRlo;09-avVsZSCI4Ok#Y>|)X8D)t84B-k(*nJl_`Kbv;ysv-ad(~h$ zUgiv8DN(WD#b&ir;ljwPBAkmn<}9UVcAVG0 ztdZ-*_Of?e@}osuDXIr93-D!7rGm1P83xoF0LuFDmk!w`sVJ@kk0JJN;A|G8(u9Nd zMbhtSh<(5OUhIAmk>ijXL`Ox|jB_=HqPsUHc9N-Plw_ugZ<`qOMx46d;`dj(#ib9V ze;4RNHtWzJK`-arrHN2<@+kLOP!wgnjo=$~v#DZpX}!nNSDYoCpoJUtwMi6nY;!Gj zRH%azuL?5kk;BEi)kPlZ0rh1R*Lhwbl_M*Kl%$o%b#hB-2icQwo%HQCxFL6Y54NuD|%&)Q22ky`Co9IK~9P73^q^H8D)X$8#|T7bkV&y%r+@NNtT zjm;Sf=Fa$&XhQZ(Vscee(O4}Vk;`RD(=-(<+1LZ)08Pd8^zGzPI)qc{OgqV(5l^f+3 z4~&r4Ibp0n;KT9Zc|LUoIj5?@4lo3uO$Y{AvDVm6xUo!y;RnfvDX1Q3G&+>k9JEeG z62*kr5B@e`TV`DvJ1ypc_ZMi1Rkt3YbFKZdNF{c%x>+IUMR}6$d0+PxHKNNaF%l; zYGThLaLO879@&<$g!<3wI0Q7Cbf{{NO?s(Bfh`w{0H`m+URcC+%-Usdn`{k_FLuTd zE=u}~fLyj1y&0+HlQ?4UY7-YsaNU@5H>yFq0V1xtPl1+87qOMHhLmCRoAz2t4K%YO z!;>0OGWc6ouH3DeGn~`AGY}F1%#T_#r%@=<2>e})wOLs3qOoIrpV%vMRwX7RS*fvs zb#od0^`S?r=tW9V(RbQWHjG5zPr_<7fw^`f;%%S@fqQu_%iA}^q=Om@E3s(EYWC9o zGeER;n9nd2Z0-^%(&%{W3>}Xhfjym0sqJ~wITh3@&6)+b81!^RRW6Tf05V*_DxayQ^<9Nmwq}6mFg%UG$yBree1z+*Bu!{)O)F`#F(9w-;v1} zu^j4vhy3@gAq2^lKnCqWD*NRitm=mm@@qIkL$}o*>BRhVq*m15iC$rig*yj}(RF{# z4sgOmTO_eX;h-5=Dr;4wq2Kq~3Y`z-(jmEfkDsfM$-@UPRHIMp~q^`TITcbs%A115q;vOeJ^Gec8YQ{E>3Nxv{*58<;Q+&E@;zK2k>M zXOplHh%`~=1~5hUxH_-)y}vIU%kf!SU91e|0IP$4DR$PtmiN)}3tFBmD z9JbSnH};L(i9V*xfK`zePBoBA{P6=ySJq`|+vw!?hPPw1@~g@VI;DFzWi+ zLR9~;)3c_!)#u6<5VFr!$jXeTe!a?qvZn2vI(eQT9JBbZ0!C?J1>PgcJEbnD6ubBg_F$x$Ltzp{johIj8 zTc{7bex#!cq`nhmhUQeZX61_1l7h@dBfFJCl$1HL2wl+PgsD@V@M37m1%7SsiT@=>}`)LGHLZ>CEOz=^*PgC8obx6YO&XXwwe?T z+V}1KS{v(TI($kAB_0LNmGO$IK2d*Wds(K=$y$;XN7WII6f%XiBpZU*HiCEQ$Wj8U zjXWnUEm@`!I!I<;k#v~=JH4eMQoZh|k`5my+8;AH#RjSTlfq1e?$v%Z*oP@86N zbu<)aP)L2+jX)XvU-CE7W>H-;7Q@Vxg!{szOg5Fr$4+$sYcveZxRTVdTBzNg1J?Me zz6^&c&sCdW*KZJ&kSHI&M*KK)Dzblp^7=EGGRvnf<5km+6zk1!tJ25G{8H=QLcyDS zR0B<%s>=1way*W<4no!jPg22Z8RJGEp}+t$p@Z;*P7GJR@Kx0P5raEDU-w-&8|5D* z2~1UjuSXZ%x;*ruQHd7bai1kh^}$MN+bl@CGSQLKc{o-z*OGh<0h%Bdcn7$Mo0c5; zqU}G7{9y@klv5HYNDtg4xO}Z{Z6R*J7*@vvR$pK~n|b{zeH&&M?^t9zLmO;b1R|Dl zy+%M;)fW696WEG@4$28h4W|haVmQ39{J8*Hq{JF{?yk zg(?NPp8j;QQ3D-)ZYD{Y2i2ma>-9=;$AF_Nq7M~eqzY4Z$E?GB4fw~A^uTX=cj_V{ zLlp>N|{&MEDp$TOi{i&Upb8&rR``U6Zw0UTv3IHRiHU)-RA59y^6P7 zIueid*oyX{*ySdNh6&6iH;hXI6xESJ&s`9dDqcjsY9|P21Se7s*yiHG*mUMBX<#}K zATUWb7~`z{?QTN$I}r9lwT9P?bhlgPTXFB7$Q*{mWx6v`;S(72GY!y~x5lm_wU+fy z=!IJfr4am#;Wa^Q2k_|nuF5w|h|amIK&o5O({Vy-rimCcd?25(x#Zi35>l>WV+qI@ z{rt6ERW+(;iSdyMGJ*Qb0<)GUb?lVAM>>2ksV~4#iee@(I+Tvvw}CQJ-@6&5oGFMDg@wU35-9B;(|?5$YO&;gEzERPq4zh`$OljFnuCDKyB~g; zf5=K9?k2s_)vr+jSN*%b{D3o|-W9J~fWVNs^ijKGzFe5Xw{_YR&B!0BXb74S!F^i zJq=z6!+YN0cyQ+G@Q|N$U5UT#=u1})`L~t!&IArs8&yKU>v}D$8sBr|nk-~b9$&io z^!n{Q{M9;yC^AY19JeLg@e9n5(}VR|HP0~%eoLi0ud7S&Sy6o@c9w**N*^cArEdN8 zTOdSnKxj!1r%TDi>Q1s{UkYItH}}c>!ksk%lF4_^S>BQv5_4A;C2bL0Cyo)66C)>n z8Dw~UDgf7=Bf+||^dNvVf8;yPr|cX&E`zlyG5k__2JWDRZBabWfq4a%d1?=ON|d_@ z3PkAdgelFjG{zFIrOe*JvTf>LQ6O_eU9h6_3A@W2)<4xjJs736%5}c6LEeRsBS@$% zPN7K5Gam$t(cf#Y~*T?ifZ$7p-0S;zxKZ5U0agY24yX0YO;T8$4_NwF}Rx; zSmZWzDAkAkTWIMp0RG{f{x7jDAL=Ep{{F#QbOF>#vd4LeuDp6U9YU$S*Jy{AM%OKZ?=J8e6BM?*&A6t&c~&`s0mWlG5TIR#I-fDs}Z=WE#gToX9_cN4mz7-+4@ z*_FuHLhpVtki51`JhnqNYj@c3RgR@b+1Upf3mbcq0BrKo+GtL+s*1q!5j)V;>=lGQ zv~yJTgYsYMU)?2-Cbw@)PYb0)k+2i1rSf*>zYV2y`&P(nof@=J=t!G1*hh{$C24?GCIG)qx+$CV}oYURUDNG;(V4?iYKLOHIl|#NJM7-6WPad zV=brNK%*h#O;V0p0*vxfr3r&}au8(<5!ri#Nt%XAy`KRsv|lWWh!eMcx^tmfuV)gY z2rA`F)6rz(bxPUq$=g?r%i4C+OV$KNZvDin!g43I8cMtK!fCu)sIf%9QQ7GCKN71d zPM1lKo}(5v=Zb3P9F&n_2~15VT7pelo?fKj=3uL#lzzo59F^GCY$(+^;+5VA`npEH zH*dhASDhA&_ouv~bT74(pm4$_?+d_+nR+T{QMwvQB&`eJ>v&&y35?>LUI@Mb?HyqE zWj1JV-jtu9*0od@y*oFM8glpAr((mVIh9;7kV8~VXEK)9qC(j4c*581cM}wX!hbx$ zbk|cJM6rRHC7jZCmn5()LR1C>j7~mla$A1XgPXi4o76^-m%5nYq!vN7LH_5b1o&Um zVlfmJF0xV59R3FDNE@u;wEX=K0arfDb)L(n0YovW8t#_{3;`*nnlwc^Ko($f<=+Hf zWlen>(q($;osGf4*cj~ z&A3CJAG^Y9_k&}nucb+G)IzjwueP#tRB&^;J*s=D%>FUIh0F<$?^W9u%XL*oH*0Wi5?yXGFESS^t-tZkJA~9*G+f-;Mtp)N&HpSf?T~H|< z*6r^nd}pqxe!hkn#%zw3JR?TF23fJ?6Wf;%8SCj@|Fm4TS4D5~JWZ&>19+*l!Anc~ z170c8B7$-``Ngg79z>@{Y>p8nxN| zgOjljxo`#(IHoHm83p1fRuw7=?D6v0#4 zQ--&+`40Gtoxr&Ci#DEI1ezXNxjaQIZQ`s(%}4p6K}Jgi*Me(_6MMw73ZyNjxv9?% zu~S2^rm2hkp`Nx+O)sjcPb)%G4_${f+LO4+Y>>Qf9bEb|LMfD`Y2MQ>98T?%oqH6a+a~ZBV>BEV7~;~_ z2YK$h!F_`?etzXgdAT+Hvt9*1f}{5lDB)9JEO+`Lrrg-d6#inyR>O^4&4CnIf0MpR zn#=jhE(N3|P}t9SQnrZ{hO^Mef$|XZV?n^^w!nweXNxlSw(v#s9@Npjx57;$rl?~7 z=*P>uk2%U%7m`FVK`*KvyO=z8Z$@yjQm%&JU3alkTQlEl2yCg?zjeqe!=(l*)+I^& zckCiP5S{ zM~XigsAk&x1$WSgk;FLIlXQ@$7|CU-W3HH`VQ7P#M!ea~|3=Xg*<_D)0$w`=LQM=!-0OcH{Amz*8r+D>!!UVOqYDMzV&Yz+WQ6eP6h z?KnoF>4(mJ#2aPcin3pwrLwQil6$bM4CHqlzrQ%iKL#bm|H9#X-zAe96>q_8Q@m@h z;)59feXYoun6rTuLTdRV_rul6^Ye27$$l%>*VFBN;a{4Wi2ME1d8?n`$Hsu!7^C6P z{z$sG^~c8WY;Sg6c8_NZYvh7NJV)POastu0XlSxN3Oh6@Cuyu>76Nr#V0=3~Ezx&0 z{bzsY3|q4pL*B5?nbw)*PMIx(aYJH5qC24MtF#pI{!iYB&==b123T5B;NZv1^p2ID zZ$;D`RDTO^pxbBw$tu(QW@-@9C}cvZiI_p)GR{5*aaSdPM!5QX`*HJJ=hI^DM z7ejFH*wQ}Wz%otZ2h0APVej3Y zdN~W9zB|>(H-$hArPA}V%IN2&yIy7bSi4=(Neja=(8ze`dPn&D1H+?iz+VkY4sD6w z5Y408CVxqD`Sec?o;|7;%|bf*7>+7ifR1!jneD{pNK+Y>Z1k3+Mb>A1168yyeUyI- zNL(Sfsj=_yXV21~mKvRl-IOkgt{ToV?v~W)x|##Mo3G503ryxl)}Tt<{--tw@q9J) zJRrWJqprgG`rjSnb$cEF(_??{e^Hp5UZ{`KS9zq{_AJ0k*B}!dw#8kPUXFn1v;98G z8w;V)L(V&);Pc9DB%nS}kFy6;pZ}m4W>mVkX3J5&`I||CfZh*^~VAN}nwn z^J96hMVV{mPVEkm=qBgOY63qs%%mKiUy04M&T|D5a=r*sH$Dn)92#owTd{t+x8nk@ z(G=byP%^W8`m<{vvwj~%q)U2l8#BgU`Tg}Lf=;F9{?-03@g}+mEmquW!p77}8U3dy zjmG)K*BgijUs_R2I%R=R1sQFabxq<29wFU!A@%f*ZvLT}v1ZMPIyCH0y6FQSMH5pD zvvL0}!G49W#u5dk&Dlg%w)3HBYnJyFB>)D4jHmT~Ntn4iU}fnY$UDAu1IkJYtOdX< zi_04$gEAlAnP@E_QF=Cc@)CfJC4;ZVk}pKdLLzfR3gBct=b&{P*jOU-Xc{ZJupt_t%%h$wnrf_`IjZeJz42I-CEa}tB zgLu&haV*@AUS5Ialo_RyaUl}!kOA*SxAzsyJcfy?{22`HfClOVMoLVwNVuct{kHP~ zKNRoL-uIt#A%F=+ihTw^+{~x`F+NvU zz``1dA50`y7W3`V1GV#e*Vw1r;QY;gIHazPo{68^yETRx%7wt|uHJ6r$cvyLzwgIF zxnRt4j`7XQw&1zoc5NPD>#p<-_-9HxW^XIjna(B z#X#98t*~u^V7OKm)Le^AkDYP2)Ul`#rChu_FD;@w?=;^RBp*8t6{Tga3u!gGsl)zcg7Qah+D}++Msi9bzD{PH|CghaX#g8= z7?4;Jhj3Sx1YrPX8(r5vyWD840<`6dtJgs`1)Q2D&r+g3DKTq@CSvjj2I#TRPt8OV zP7RWrlI+cffc=DCQ`ST!>mJl^)bZq<+y%1vG_;tv4CSlfR2;d}QW!r~0uhm%UCRU? zA9&k|=?5*!Mq;@$WCKYGsbk0zeqjuP6~h>VvFTVI=l_}YfE<+oo7)z=_41v);>>T$ z)K$9wNvn}MMROB3EN3Poux0uUHs)DXauAh4gC*HMGn6)eSudP$nF1cYxR4_LjEg8B zT8@~+G(y8j4zA81T&iPX2bNY#33Bgs(U3o|vJ`X0H5v`B6O&1PNU*kvE;Vm6H}C3J z08O3mqoTl}cvRgsa)p9+DteiKT?C;uc5<8zyg-wci&9jRNr3WDP?r3ma4gmr7*<_z ziaZsR{Z&~yps}7xkqA8*gY%pwA2g}KNga+x^f~tXJnID-U@k8^tBQj*b&tJnOA7fS z*Bf;Nax1?gO9TyB{t5o|bkPAUH#HK+Iw(u^0;xF>$}F%o+&nM%li0$-Fbv3HgmhZ9 zSBnJ%2_78Et6Aoa!qiI~G$~rl{)11r-^~vYL~0}_3PMf7^cIqhljWHZ<~8HVoKY#* zB-;D(G1hinzNY~zO9e^+fCMCpKjFbnnkMROAwztNQ;WpAri~7PWBAjsWrc9YJ1H<- zFeN)SxD03bAPzKRj&17N0|W)8CvM)c)FHIQ0?aHWo8s``O+^hZphXbE7#7x!-USq3z zraDoT_8k>3-$k0#iFSxG{@@uaG)X%)(6m9B920$xn= z^h%VW{t9Xvo7Jb592;@&^RCD~tgTpl+8HMK6@-p?Z*KwWJ{4ZqjY{F8v5cn)d8u|d zwK~+wt7;i}WH&4?ZwcXqbaRrd^IQ&WhM}Xj(E$=Eifd?-DDK0uNfm*n=^rE961oUt z+6L{AsEsRKM?239nBNQ2f)AV>(F=&fnj4u=vN*^3Kk07wRqKX1?NIZ242Y?=SIWGsYYlES?bW*L&wWlvka$1RxfMT$t0 z3t(0|Xc@dV7DvCB7ETL;Mn&g!vPBh#llpSXfDth{LnaOppL1k3*^ggu9EWW$uMj&C z-!_+}uSQ0bYEv0f;+EqIhiGPMYA*6W9!9x~L)C&zX&t*`d%x^8o++eyJrD<2-T4GU zqq~J=Vj{3Fg2s&*>$JA;u$t5A??N2HCq^e4OtoT2g1bqZ!yeyjDKybQFHZ(1fXp5$ z{EZ^ALrkd-k|#H{8{?%C$B}G*UvBoJ_ zIb-;v899X*B0zp_Vy+f@Pr91%#q7WUYXR2|5z)kLF(vI_$IE#i3k8Q%j#h##j=Isi|_kp9QJ`$_u~ z?1YCL`?KqRJ4l>70Ky zx&(HX;=Vdds=}=FrjW2i161Ps<41M*14#;c8WL_8+as`)q9e?@I$f%@oswJyyGy7n zn3g9BABzNJ#BJL2KrR2mWc)95;!qS|T-5*i{V6iZxVW-=2S4cu&>1Qd})t#ehCQ*Z76J91we20#GqU! z(VN{>V^yN_?u@;j1qTie8%WQ?HZ&mQhsdr0GfS;X`b*5z`L%1~;QAD4$gPY|@^#2N z=7Gc1aLHH{b@F5)xc5a2Sy}j(327&M!xCKB(qDnOhJifQ!uP6|?|)lJts_)M~8ws_T~K|CUn_OropKbvChP=>tk7%UlNLEI|Q zMkQo78O+3|1Fa%WyBilRteo65^rWqLjiZD~RW&E92 z>@2KeAlF~L;WETC@W?>ZjLDI2*!9ohB@=h-lce~Rq?^FVQa03tX?{KUX`} zpRCBx&wnK2gZkC!;G9uhAHm;O^*#RH@?gk zf8b@%+SHqSp^qHgF5<%zenu7O?}Y)^EginQEc z{R2us6!8aia0#Ud!z}>1|4Ke@_y(Xu%-BFyFf31$iN1*z{#P^gy8cn~@$>lPqz(hn z>sJT3|2+PGE`)jad;E0mwl$VS9u;J)Da3RcN&E>eHlAqwxqp7TIuYS&^V4sfJA+Ta zmkik=GW^Fkp`b($x>?rkC)mfwBPm~b^#|HNJ5i#O<9IhBMj$b#r>B>vr=HF5?{G!< zToZfV*U^o55_Ht>t7v7su9fQ3usnT9Fsk{vnZGX#C{Mg-ul)9*^?YNdw*epBp9`L! zD>)zCVIO{sy|K5bgXJNlJv~S&2ZS0szWnZY9lR5hk{XWxwan1s{03TPU?;&5o@8aQ z+$AcwZu{-w?h<(yS66};SAu?9+a90Z=Dd;OhFGDU8@sJubDxbUA1&JpKEL#yI&0s&a#yZr zMBQ}yjVo15?CCX#8-GSx7t5*3LOU@C6`&;yJN@wP&28>{tX^pcY=o`c@aX@8-1~Lu z00q)@^+W)wyLF}u?&+PSo{Z3i`0w^6b%AaWB#5%}Z?g7@PECp@vGGDgTO{rRFej!e zxVnbeqD>@)v~0)8V7}>;38Sr|7dROYBkg7Ub3c1-isy210O$fH%xC0p2HSInF)qFG z!q=RD3#DQqm8P5o`DcH1ei4qjrvRjDG~Q2VM||fQGdX~Cjfa#*vk0zUUtM*;EW`c* zU14?w)z&?buDMz2MW<6Bz_wYsw}m`m8IU>)=8JOTopsI2dav4*hgAT#sw!XPaE_Lg z?lk9ZH4+yUa!QcyXcVVZpBzOu$O{CuuG=xti0hd)rV9BoAi4N$@ zz{%=YTvTwX%ibYrb&@b*`MHp@%xx&q9mRsjg1!N>#+;rV2eQ+F;ie5?shYOY;u3L( zXIIe3c;#YEIret$5t-DsT)TTm!+yiQk?P<(t@(S~g1(v(SZ&(g^g~3B6F13(35yo3 z^YRdnNOs4>Uhlg)s=^t$f1>uY$5G`z z19$yqH&K|!SBb5ab}xxP^IV~Io^eX=)kVq^`>z@AKVVI!<59G65ONw+QWulU7z~0j z1ti#{2JHO~S%S&>d!ayp^fou;htns3^Dr9R)x{u>Y+!>=`%^d zrIOj7Th4OJyI$LYF`Rd7G6M+KghjYwDfeg;0KpoUK_>i_Z#51sD5G#!VT`gwEaztT zy$L-MlB`ILy1lAgpULE=VzIg8c(f=?11f2C`X+e;zob8^KZ@a-KEVkja&?CESCL>I zC?2fF^7!R9p$=a>+~mpRB43k%U`_ddz#7*c@jS9wAXu}?Ea~zf)}3^{+SuK;b9A-) zTCan2-jK;P&BBF|(>+&86l~$U!7duT{^$P|%v=e&i?;8NG5jCF4EL8{CRlY>Y%nx` z`%5s>t^c+PB`oT}yzlLgKYitqc#Vr0(`m^Swu8ig_H5e@dj z+*_jF^!F81&)&T>#7hrLKSXl~n?~GK83ikhTV7{I4uP?o%oyE#JswU}ktx2l1lge6 z-@Hq60r>!<+?{j8j619hr46H=#Lpksj$vDl7#@#MlRG-E5wRxh7wHeS76EAS#eU-(MD4o`s#Aq z&ad+VrQ!P>>NMUtTzBmAIPbIz3uaevCdLRk_bTjHVaW+tSXv*htHYTOgZddScxx~A zRyzaY!%<=>B#ogmOrm@Z`gXPOkpo;Cx-ULIgFc=FeDO1mM}m0p_dx&X3C${kPsp(y zG-+1MnnYyTYs~hdf6EBldT(7J?j_UA)@OB!1d=rdp7MMzDwbbl&AtY$Y(K#S60yS< zSu?pW8_G6aJ8ZlVpi+Dz>FkReSA3czW{e}RY#BR-%}(qo?};ay)CY9TbOOm5#+8yR z_}rsD)|5L+^Q~t&tlA#Keoi_A^|_s_ zRX2>1hRA5Ub{-?f5}acpUclYQ(5YWy!R~3f|H^2XOfyPU&)eY z^b(z@!GVW8X_elip7e1I8R(8$-+mhsV0XPZY^}n@7>50|w~nI)PfRo&1i3!F*)1*8 zQikY7xY#h+&mdAIZo8$uP@z_{;N#5|<;)Q>#83wpMY;|3qE+GHS`b_i6$%F%yOo$K zO>CYFsTu~wvY502d^$H0T=t-{Nk5fCF?5uV94lQrJA_S4v3xcJtFpq_gG)?H@gs|> zBfLB9+R^v^)YcR5v))kqoTJSV9* zjc0(T0u#P%ag>BG+B$Q2Ix4lY1oA5V>dtE5`*EF|zgS?#_ZJz0n41^$N_u6=P6s-a zT>II)LAsu;ZeR9ndC$ZHI>4qc&>pNYKd>RjHHF26Av3IfZOlG5r|0fkmkkR+CDvSW ztmY=8^YiiLJ==YG{nhPsY&W}Ud2ZEm2O|bb`wd9e}^zo@81p7#%7RU-0w^hbc=q@wn>ozC6A(}HH?z#xzy|0V zJrMu!V)y@TXDWqkWMW^IRAgA^D4UmAjKkxWnXj7Aah7~5)NB)W$e|A>_N*&J7U%#e zzdDkgV#)Dy@bVuu)G4L- ze!!)dt;;Z_lHAYL<$d1H#rs}Lkg9YI!>3234PD`lOaNr}RK9pgXz!eGDXP;0$zcu}HI* zIr5AcIu4l8HH!!*p;)Tf{k|pSvVAJ@B^oXpvI~UHA2NBZH=1T6P7IK4d=3Qs7-#iidYWxfM8Gvl`__SKN3 zJCJ#xOO$A%M;~9Pn7|@0>Gi3w{H0kT9HNA?1}1!cdhm7t+UzRP&h6eSyW72q*lO8U>#9%j zH!uAHW*kSXyzlFi{;aQnN4I6aTh*6irqK&grRy@(QOBuGFsExHJB$SYNjA?Qh^ogX zCcg#*Yldj#s|LNpRA5_U^S>HPYCUiUF%_|jS>tR_HJ%>PEv64?R48Wdb)%rCI!xDf#a=$N56@F3_vAp?@D9B{sz z#e}p;4EZ%4S?2Fr_V^>G{|q%8?tBM1VXbGn4m#fl?Xr_0BY};j zlC8%A;_q&#*!F| zSXy2ExIf8$9Ii3>AB$`>Kys)ln_L4N&@q!;HZ1cW#|+kD%;b#yk6lrm_@8-t1i4+j zZ@VWCyK5UkoMCxc2*f8>bEST~@4xJTikWXW`5+4x)!R9r_jluygTRL=E{PHK8-eRd z0<-JCikaRB@^<2A=M08kj%Z7)Bs=lfEO9~DZPku)1SdzvBMo&N`5%Btw!teh`G;3wH(Mqs1gXlw&yE>%@)okQ%Os5+N#diB&lAYm=t2Jlbj&c< zCg3Bh&;E5^5d=DBg1;Oyo?!@)oMxDozIE}Z(@6OG^u_Ni`dngkUfbNmu^qQvYRn1F zo8oh}x>XO6sZkF{f(}0rJ;scFOp=yjF5x=@!5W=1AXsy|+Rdjy`Jht5OqtlMIw$J% zTU*pt$qs74ao9rXJ~dzWDekwJVUsmCcoVzEcXAr*M%qWW?8p|?!&3%=@w@*-mJF>G zkt(GwQ&k;3hJH)_$=(9(c;z|jC5I-N{)#Ns*TPL^LK6A{zMPOG(bC-C1pDYv|@j}_eokEK9Y%vG_|I+|P3ut2i4Bkf{E zU~?OKi+^X7_7_=0nGmx=2ABqtH4>j*k5zgAd%(Y-cQ?V_S`vWdQ*Y?c?N^OQ9KHvP zYn%DKvM|fs@6!5f)-~N5>|E2$V68q!;jKT0*+-YpTJCGsAil_&3#B6qx9!U>vgVFS zxBtNwVwS!1`>XjcoyO`Lw|X8@O$>LSV@B;slUAjqw$r1Edu8$T6XM}2^_oG$<>gtK z*_Pbm_;=Sz#F8r3*;i$$^Q*E%kHnj>J3A+ni)pHNsF;g5)|qzlRauHP4YSQ-p*+^{ zT)m*#?ws@sM1c9mAmotnzT&4A1z6qO+Vp)S0&I?6+1(Jj;&hl{Ns!9sDB~?!Glt(L z9_{RIRNi4{C^*5mHWK0QV;b<%W0PvX#MNv{3ixQj&q~~$#EbY%h7ln!+wIJJF7?LXex|dj@!tWyLk+*u8c;4eS8j^3QWRi{c z&JS$K0{&Wqt`$f;>}dccGucexS_0(1Mr_8qN*RfRHJ6s{F7}5x75rWz9chrj9l>2u@(hmfd`ytD5RgrhFaBu!E}9{(RlU2mA;vEkyack&UkUs>|@aHfWFlBcI(Fh zd|Enzs-b^+zW9z3x+*55zp>XO?%tiWko|H!{!sQ9s^7ib?7n4Xnc$iXA^s&G6vCIS z?1Ak0$}G*>C%t>NOdS`&Zhs?TSW{z<3EC%h>|iP~Wg#KaE;w;1nvkb0<7G(HcN+tg z?V7UhhLwTHA6YnmkSmep*(UtFHy!|1mi#RZn}qz-WeqGaC|;Ws4q5bby9WLvndyWc zV?Ge2G<8r839kBG9zKyj187`7Vm-dtn*KOq(jMEVD|WiibYBdj*a0%~&rat5W|oeVh0OP_ zVS$;YzIfnw`29bU83BHb|B=i@vBXiuaJxO)j&L7u>wA5w{{{4?mn0pKOiFin6GlA( zw(R%;$1|TZmxG_Zar$9+VX9)%RW@O)QBonhY?1{u)+pj>yAY-Ulr&E)bPc1>xC?CL z-hM@Tl;aF75|RsbQZ-aulDM;jbdNk_MVJ~&ZbklNQf%VeCxCWr1KodoL5$o|#reR@ z5>RS;3LG1lkUsr2CjBLudAv;?k{s{aAu0E~))(D|3k6Vl0W(Vm-gEn3nWf6AQV@i} zaA!V;vBzBS^N#CW@WJbOFH{w@&#NvOhAv9L?ys@!0haO7XuNJ76Fr6G7kjpfE}{s_ z)xjfEpkhy_o+{4rs?z?F+?VzekTe*iZjTrw*(F7I_XL>n^>Zt20ldvV63hU+UVh;W zck^HUSO!ly)*h?N?Dc-v1F|)BAjGgidRHXnBMZbj|6yw^(5&@YwU}Ax&;N(55m!%S zikki(wk9{>4-HD}sMn6iPpU|k+a7_s_OVDY{gmWKhLO!c)~?(G?{1zlS%sApwkiuW zPAuE+W;8e^GD)a-_XMno5s5qA@fK;J9mPv+LSg=R@lUbVInG$_UYF<<9_kn%n^kM_ zW0Po!OsFMvX`b@B**ca)(?di$5{VN(!zGIi*^U|u&th0r>9ke*5c#gmeU-@#L(^vI z3ZD~za1E7Yg#r++*^a#h9!u1V@=SAenu66z`%agV70q~)`zujNyWWLxgjQ7PjNvLV z!zkGB<*Cs{l#AknI&)Y!pqy~EJ8OyEv z){hzDT9Uo;&IVo>@SbqiwPhnxuNR>+zF=ZR`GJ%Q}}Uj-*>r<3>@MrUgso6^5a{lV$zgw?ro^iA%XC=H@J9yy3Fnn25p?5qAmP)@hsL)QWg9qlL2I8JTr*xNoI#jTV`teAU% z_{0ny+A+$RmrS#fdFd*n!)D3>HWA7X4A=q#@R z*}+PgrIu|1rTh@NC6B#@eC`hvC8D>oVw(w(p&p3z3-W7tkJXoh^+~W3i>Usgr@=2|SZSmvdlI+< z9`XPyZ@GLPw;Z)6JL70qUQQ;u1Y#GO41}}av1Ay7vd(!-$V}+sP&G2#)l67@QI>iH zVlyOJSYT_pxL>e=fQ=LkB62^{Q4HtxsYcuwT^`Np^sQw%!%~eZjs37cVxD~)7pXnh zajOK(X|~ND+DymDTSN*&4ul04@xtU#Y)q_0BF+~cDH^){93>ARVFQL{G8Do>cPE`G zWJF+Pb{hZOV96%F7h6KEG6#X>ZUSl_$iTG`I_N1vlpS&C-lK{;fKj0O-3BsSDN;?? z+vp%rEm(EoMCl*Gw?=v;q8Y_pW?*iq1eja0jhHK1>Dwe@{#L_ol)x03*D98Ka|k4C zwzO)T7J2N6RfixYln+;)u$z~?uTSq9>rMo?(U<BENDtb7!Dk-3J`d*vm=l*pA>siMKyi7_wj(B$J5 z!rbEl{^SWrK~57oIdG;HyZnPBVOfTkI(-OYI8HIR1VTB@iCaL}1?1nze%xya;1 z+^bNRm$-0%+Xc`x)1Jr${3O6wd&x6}iwQpPU$89Wg$%n+-c{5xGzjwLg) zsp~zzY3f&Q=~wv3e{xIC3{pydlui*og3mK2|GenO9im-b^Y{fviP^1krHdEg{tSf& z$=mv?(6(;f6F=9IZJv`2Mvo%2fhsSHiG zhxIf-0z=_A>PqFZseh#Rm0OYn=9Z@4q@cYtS=oJ={&m7}@2nJQ#AxUH8vso+G?dIFortYzD$D_7pVi6#jfc?kCkrv11H+6xzhks0 z5V7F^NwI-TJTI7n9)QK6Pt;i=+p;nuuc;&5@)N$o5{jH9JU67BC28kPc8kdNo@o;r znT(?5dczX&nHnn~kwH|<25CU@S)OY+7?}HxB_!FH9yqM7(e|T*AmI*UrNX(bYd$jA zGFP`R1W^vyTWSIJmJ}I?D1f~sM}&c}w(KG5;CsxZ-^t$?ho!{Cpsy;;t>o;+UP@;J z%@7Wz_^qjx=}K9tK!A62nElld?X=b-PZ3hv4B(yVQ<06_3ab zwNsT0vKj7oF#MieF=3G+X_o?J&@yc8yDzm&xgHd)3;{>S-geU@kpBPJd&{P}qP5Wy zCwOpocM0z9!QI{69RdUh?iM7tySux)yE|+M4!d{G+qZ7_$F8pKPvFP{)e%lLY?F+Z*&}ex!?LY`Rg2^7vaY+PQ9GyY=%I2c}VUqO2Z}hjXsKZL@*GG z)a@&h(K7GI!0!D?kiJCyrY#tp+V4BKe)UbEmCjQ=Q^r$}YUatd;Gb#+HAm&2Y6gk< zd2t`DZ*4xK|HX%q)&>*m!d!GL=nbHk1!8P+==EW5gbnf;7%KlUHpXH9i?PW8F*X&U z|C_OihM0bB`}oJ$hzE>{z;tW!AA=YhZo-&Di6qC4s^M6)@v&kem{h6nNaP9%RsO61 z@t93`l0MJ`LJ8uNdzaX#dc_%8lc0*FxanHgEF|j~oPA8cpG%brvj;BVyS6LI|1dVD zXzTqPF>~kc_?H9{7BfTsvZrgSna7OAesFR3S}b>olBExsW0VZ${NBw20!C=ltyadT z*rIHi8mg^_2ks}=u7~*$(qLXyh~_4g>0sXMnBJY$?jXnp>pzeU=g+{EIu@XBCA=m0 zevMQPU)-NeOtpd=$^Vhf#M$8eg-T)CM;M|Op(ULr>$Yh0Qo3T2Q!h@OaW+Wsx@OyiOL+Egn03xy!F-BW|G0 z9|_MSEBS&6=m}7AY2c!ynFQ{aK@^{Ix^ozqK}~Q?Z$>&!Npat($cDspmE8h?RsYZF zPLR>-d?8~tT_U-zHtW}al1n=S-&|-^wqF-f zY1VRfIlofTpt^<(mp*;E+oZ~vc0(wc1iMBgsg%VB`uu;%B^9=JvDtZrKVu11MT>*4 zcoL(3sJ%(mov&1uzR=@)$O|jn>EJCWCpu^7b9u;cP$HQ&rVLQUHzOrbs=%!eH+>+7 z{>L`6vzFh|EA!7bqflu$jDg4mqnoG`U*rzPXg977iPM_mqgGPb+gWt00GGcqoea>ytQ-{tdD zEXL~}WrH;)45DoE!q;1c@q_9>$t7#71vY5Lm@;nO>4VIDB5P1`2`Xzc$o{v!UGw+W z0=ta)B1LL3_+rJ`8JB3ptP#=;vR^vhy9E)j(J$I!7v^GGCaN4|Hf-rvk9(s46Bq9C zO&l`{jP(wTwmd?rsTlVBD-FYYzBn%%^I> z-CuvoDORxMBI+P#b5AJp{r&cASHBA*?36xxRXphO=RsJ1=+2^@*w24D) zss1Z2aVFq>t>4TXZ-a6ERkkTDsS~}qILK<1xx!bT{PJVCwQEws=}bD!Xv0e3EEJ(! zIj621v2Ph-&OF*~Hvy&LytcCw+;p5`%w*XV%nUt((XO_^DY2Z!87rO`fQB8?8rRCQ zQJ-xvk6|9jokEkc$(NZ*X{hiWWnv^+kMtj8^P?rG76jQygCHA}SUFzR<4_r`Ti4hp zc!S_C4ihTn1Kh?EeYKOu1tHNme)8YUUox@k9+MI*Jf6{_VaLt>)H2iN;S!``i#gD*7y4TlBJkZndoz;kyApU}8Y?4YoizW=SVtjFujK!|xa2Bh$@M?Q zC8%pToM8X#LT8Lr)=48A^+q#Ah<)#pWScn6tH~~yTumf9^b!c9=UpB!rmsE%xGu`K zmQDeI&r?r8p_pOzoZ`mIE~vO{ZyBPKcdQ{G|He+u3(!$$!@Csd$1EhxK~}%z9V%{& zS^VC5%R66CZD=68 zgz^vJVr~P*6nOsS;0y#lJ-sSkycoWHbp4}Yi2p;w(1!g#pkZ+SOT+w^hWRfI^Zz9c zQ}tgO=D#$||K~J}42Xt#^ZQ>kOnvb#^y5y+8{gmenvRZ-57@UG|94)tnHX4i-a+U; zUBNj8Uy`$}My(sva>vlPc4~71>A~1gx?UV>!~$gMxqB=QF;o`-eGfTafU1NnAqX$RpZb^HZ0<-TZgDvmipx%-iQUs7QO*iGgD9ry)D2&{HQJDXtF#m5*82SyM z%_mJIf_2(4Oc+xHaND<8vjuqd>4>6}UhKGK^E;jYPB0eEr zM-rVaaJ4TI0Tf));K81RtC!1&8naO&1G-ijX(-(~GLIk9$(;Q>3$XSYpwpE5ygYp_ z+B$tcO$ks^Xn=hDR#4=mmR0N~Oq6?D0=stUEPx<-@Ie^IhdZnETYv5(IL?;!pJav% zU6AqVC5Hc*ed@_};-6#&BRR~N?hqx3$+5T+Vx51uWVe7`%%|Nf(G0S_O4OTbZv=d8 zXJcdQ{tGTEzv(hmb)oUQYU;ECfIw7x_DF7q@7M{e7sS6va#z_44(Yj#nbOru3ZDYdPkqKD87kE(&R1VnqAr?&-KQ?jz7>nwZ)tqC{dEVyX+2)+1xCo4RhHmg17SX;$LpbH#&d;P@jJk^xUs5on23#FdZG>M8Fd( zK0XMj95|4G!SouPt+0B3Z5G>-icideC}dTB_p}uasEE}SVK@^#B}td9Vi2s$@6fsV zeNHb`!cpIcG!i?;GJN3HYad+%?TLN~nqFT5ge@BDj_S;jF}CUmPE6$6wkyvJE!(w% zX{Q;D1v(Eag=w^G6DKCB_fRZ7OTCM;H!`z0YW8$-YJe0or2WTUnZ;hPNvXK+g^%<9 zQOrQ=g)q!p^F!R^@7#0eFNGx05(Vt%no@q~8 z{GeTS*M4xv@bLbIAUe@%8)xNV?%;kHY?6!r4w+9AKy&{q<1pALfHu#@o_~aavJ$?6 z7C|Z7nYuct)mcMH%1&4>S`FhPYlag_X{;qP{=^%qAgL5iX`R8~SX)9!gGygm|DJQ^ zxoMj>dP#2fDE{;D(IZ>IwtZoscK7%r%n@Tb%X`f8C47sU76~JdA9K3Qlw;eZ(iRv&M)T1FptI=gTk=0*FSW#$Ak5OikOva{3;1a{c zT*I<|=D4XpX^|$ZvdpkSiWzR@sEq$8W>`w`)3SGh`hT&L#0hrs^Lu+f4$NH)G_=4v zA`5Zhi_Y#8C;~m6FE&Aj8Ss4($l^Jj9=?yWlccn8W+MCOLh5dhx-PTb=B-SFfr?|Y`hgT71B?{%Z0`zxnn)= zQR(5t{o;eZ5O}gBCt^u)G7_%}@fY8e^d;`uSPPu`hS(6_XCGj&c;?><3B@i2Da zp4ct@VZ!f2y zDFONWLaoOSjg!q%&AQk~Fr)jZ7xEk&MFOioj10EaX-sg5^~K-VB<%l9{a`kYvW%`(JJ;)&kH;b59osa1-9{64(4EnPK3wdrRiwIhh}D z(0ti8_(Z>y^5a(gsz(7j;aoZpdVFfh6&MYCy6ez=56=#(#)X-eRz1Oh{kx8k-&x4x z_567LpWIUL7K8N2GxYx@x1^YhHSjHwpZ9Qf2S6H;45?2Gt)=&_Kuv}|Ic+y^Ll%}f;wDL_o2}a`7UwSMl1?6-oVfJQ6_xhiC}9s8 zl`d|Lb{jTo#Qao6nQT^kwwp7#FUA^8O}cMhcSsm5*2{$G0-d`iZHk=b@4#AqTjHr= zk$;jIya@#4fkbj0ntGC#DWAazA@OvL3KCL;TPAWThQ0D!s>*zgqEX8C`!uX?@=%#B zVU>w2)MI7&GsSg!saS$U0frwf_3yBitd&-pi=H=<=AFU_t-_xdjyGKUflp6F0HLjk z8gk^as7Sj)c5-~{976?!{h4nluY+8dHiANeeu1t@b|KNLKQ<1?OAD-*R(i4HA21AG z?Gp`WQeXVk_V9%VDL2ZCot^wB6tFEC9sMnFV>JWjD477JrE=jly39O6{Jc(`_pYl_|DZ!U?03U%w`PG@y3zs8Q5?I`!>_{ z-VFsb9ID=7Wj3mM=YM838*I7y>3=hjir%u<7i7GB_sL!b>-@`T6W6Es70(N)7?|tC+}~zGr+_*A)E!bL_ITa8O6n=FWf`l{A(qI%t7}KGv?DUpAn+@D=9rclmjeU${xOo3gr{$$<;o|wDr3A~`xh<#$E$_Sbto-UEzgHj`b1QKEp z^kwgTz%iFPnuXWPBv~XeZ5v!{Vj;Xasnh2bjpD*Po?PKVbb~JEZTsPhpDf$?q$UK- z&w4X1Lo3tL@$sLXq?k!yL&NvhLy76Ki=-@;Au!lKVw3nv)G>1DSK$mT4r-&VB~9B# zIZqBC<UW>l7aORbgEOY$Oqa=kZ@`OjFl^gtupc4!urZ%4)nx-|E z$TsVdJ7G3=f)F<0DKA(R9Yi0UWo||eXGJpJo^%hYE!}^g@wH^o>qdp%p)FfosI&0S zGw}hzV?*=aj&}bvGns#nU545W?a)Z&pmJ8u;&a_4-Zu+@|3*XKG-vp#`s?>=k}B_f zqB@19P04E554A$EtqLzE*_?^$43aHT8R?djJ`K9a;yM6v)vaFk`$g8$){?TkV>MRM za-m6sy$8F`G`F3QlA-}InZ-+>D3iEeV)1?0;~^XdDRh0D?D(*3SIXkEfQsnVC?lJB zqc{@oht_Oqzq;zIOL7|ep@@EI83jw}J(*G8NtNc3s}c$T6aKv2HNgD)Gi4$vPh_Hj z^B7BZS(rLc(aDb|L*Pqb*UeZGwc~vxat{ApH?`FAk@8~B5#Ub#N8bZm>sYmzr06h! z=10meO9{>cY~`zqgY*mv@`BsQq@-L>aH%kP7Yy~?Xxj_2CiZW{B%myRn;|##T;)oq z2%v#@dixR<^$if_8t^;$&-55fS3YhD&5aS7#RAKBOA1-*bmt!;NG?e=1%>@=+q5az z&%`_|B9eME18hBoxD>~E7RAWFn14%}Qe0cr8<_g!8J*|=zI}FWzN`4Sh6Ht-)YGaX)OurQ~-_BP*QgNX|D1W zEHYJP${M-?CpIq0YucExD|x-RT~yb~`Cs%f&L$n4bbe|Rq_Pf}?iZcnE;HoV==ed@ zMHh?%1_v(y^9=`5R$*=<@uU*Q;{zh= zOXW#+vo_-g;`R!qY1{4`q|<;A&w(qS2zXiazYelCzj8OLS838VAk$KC3!kM9A4*0j zl8iaygeB;T_OY#HU_mNdNbRUv$aH2=>xWxKV~c)?WIM^H`0GiQw)zxu6P?q&l888B zW%;yr?@FLjaNH&?+XePLn)~?yri=8XJCI5GBdL^^J#DdOFqu)xrln??r+4pn?#&X% zrj*YBJMkkGsU>!orZ@Y~Ub0wcW`;x~G*7z`(UlMLYxkk}fujd_=cUe?(AqQ()dSe1 z?v8lK0D;J*R%rMyAk|E{OFwn#sA=&NNHr4*`%g7fqx3n_WX>$}@Vhq!K~!wQ?+Z4I zAUSMn&3%Fg1$sZk$^QE?`iL|rxnzW{G}cTqMl>ExQu$|&Y)t+8eifC*K0M$42lYcN zYT@&HA5k-aA+bZuJ8g9ABpTnWvok%oA&zx%B$|M(|M2S&F3H=hB_X366T$Z%sLAY6 zv|4^Yix3JgPC{H5-TWn^D#XTM)!KeZ@Hb(_FGf>$5A6eGAJH0u5yw5QmTc939E`GhxJ)YF6r35LBTK zb!lL4s4sQBythP^g`ESlsB+~;6bwXORi~Kff{7Py!p$Yo8~SjaikjwF4vrl)_3PpM z=b9njqy6WanZFHIWMo3kV$7z^s#!_Q*AS^Sph`w;Wu6&o12vc0=G||R4=Km{`(;7R zB|?k;)m%FN*Ibh2{@=|dn%&CWjepH0sl#vrvt8dIQv*Eh*o8T11NN8Z4Vz1vyezwO zV0#21dOW(Az*`wZONoYP8z2r6i0T!~5}7%pB)HDTmJ2NYd$O!yaF za#VfxBt*FH87X(dBKh<=Y`gygI#cF&b!Ze}3wa8jjtG36&FvEfP3Lkmhx*8Vq{H^`u=$I4u=KtE?>)C^aIUE)KhA{Nd(rtZSvA7)-P&U^&vFL zp#GQ2f2vDu+Ii2?S}KSZt)wS6LW>dqs!KKrh!3FZl22(2LL)eiTx3g*1uLWK?0kA; z!2y3Y?R5eciJL+{L#J@Fy+_>|=U!~RxonzXHX?1hpoq`}E1MHhxLp*5&C!oIG9s2^ zOQfnaB;#M-mkP_rjG8yX|LJD-{zo@sb)#|^d=5jQVUs#y1T#e5=#LnOJ71h-;YlPb zhF-=HE}B~?|BK=~)i0-xT5(>}U)GN&5cbC|BB$>=Cav$3cJ|^cDxZ0&_24t3tq{9G zTa%OQEJ3-UCJsdOW6J(Le^lY<0uhliYa23i44bdjF76p@Qyz%tSL!J93bM(1ma9db zgvb0#(&H9o{5w9bntzU89iySTDgLLrByJ-->S9ZGf;UK6>_U!f&FZ3xKJv%f;HFgN z7#8o~kqx`Ah+IF|zJR9)ovBpPd$^npN*6&qjkW%Vl;C9f0000JO`Aj!r5y(z|DB|{ zwr{#$raXU88)T7^(?i@B-0!!W@bHmQP5#(Z%U5%`vC>KPZN|X??)}c!+?hU-T?AU%V7L)+ zknFCL`XJn-zv+^)uz*>b*|C)_T^yT)weC%hjh z3Oz)?hLs)PTVPLIZc3u*2tt(TM;qbgCA2^DvLGd-OHj9E>lP0e%VNJt->DhffuAUB zwd>Rw0==M6Qtnfy9B;?;1P(zje(w@_jV;NZ`!5^TJ=pk3$`wjCG^(>85!3 zY)38fM?)bzJqkYxI*%xxrThvvi8GP@JlaCr#d|d5S>@_)3iFIK_UmbpRq{gB`?a7P zX-tFo3+L&^)(jTF8G3?tdUdz4Al&UDpOT$&p%D@@%%Bv^nQth@0(;6)ah#;%{dsF4 zVuRh#7_|9P&~x`mUWS*jLpu?va?bvil-VNM3TUPCZa^|9Xs51R$B~!RnPp#Av1lqn z{|$lK8`yVIN+2%SdE-*qZWVv~qeO-cjI+Qxlat8jl!G6vylY}Cy%-v&zn|_$uU(~| zG(h%eIIFYiZ?-c71I3py2ry4DY{tyq5@X5!dc?isTN$e7WMaSL7D!1>rXF*BjVYD( zHVKzEeD@HDn$0idJR|Jc5h%5CuI$&FMTGPNrG86}{tb|h02j(zre03zlTfta>5W!F zma?+dDr%d!TN!jL#ckvnY}9O&BdfWo>z9U;HzJF@pA1U-L%R2c7w4Pm@jv7#`#vtc z2~}fc8!8GXzs-i?`fj=jW9aIi7Mh$59M4_CgL&}g+Yta^Ndx(~d3c{EE(XBsNoN-L zG7IPwO4cQnwZohC)y-3FF?4PvoaM6_W;9_8?#jgsqVRnaM#zM4tiG7wMEpraW2sa# z%!X#`W6MdBX^s}F9A&Veej>k#ne*%2V{tMP1xF-86iXF{Ri)AY`B3JqOX6RV+=USx zIl~`aT|lr2b9j=pf}?73`o}KNr{YnMgk*Hm3?ecsF7|l3FjWEc?~)02_-(2QkpQKf z4x^ns1sgy42VB&%e-5+spG*yqMcs9JNujm_+pgg4gQX`Q>zY(JP<5hZiZ6*0rq%@f zXUafZ(&~_SWitB6Z;A>lZ%i@~r{(e5KR1bRQzc754=ow2kPvgiRQgNvd_1$jWV%*7 z9>vgg2aqWui@l0A-LJCSW*a4%mJgb+_rfeut=|rMT1MY^4Qz;H?3-8~)880*D-=7= zBbs~(3Dc-;`6pJAjdI(%>#Az8t%9oIw5~{N!~7N55H4SV96s(bdcE1X29apVbPtkW z$TDoUcVnCCe0|g3>b`vQnUY|F6gah}y;RB6S2Km=yNU`xp#g*(aTq!AnJNGEG+Yg) zw@2SX=9xBa^FP#Cz?UQwJ<1@3%1(?8X7(azyjm3|N3e>xwRJ`8*RG&Bi%6xkq1P77 zJN`A@80R6=ChsRPfh+UU{FRA3ErX_YGhUh&-8;&Lz z0|5v+=zIMD_E7aGqT2Fb&$e-6bit-?KEpG2=6GmRvmE*l_hs5bvc&xAS6R2_=?NO%!O9bf2z{R z+YC*ql43k8*=UwH_hsUz@fr8DDWP=Gh{!CNoB!7?oReDbR?xR^WiB#4nPKp((A2=? z&e1CxoRPr_J7TcW57*hMjv#dF{wn64sXQzAMpm6l4eI(c73EH8pKsnE$fN^9!l+{# za4e!9tGUnDe)(kfs=MZ>ikFgKH`CTg@lof*UG8le>hhR(T*160sa_J8pwd)J5tiXW zbCd>q6(T)nj5O$E@ zTLgLZkc9B_xo%3q5j-oE`ZKph@~}z+c^5J1rDR=p{L&g?HDz|9m{tR!@lTN}oui1S zpQ_Oy0dgLStb}QSHWve0_lif(BX7hVW%k(CeAl_>OKRm#aJ(Q)U+?{-AhnaSNG!wm zBG5O0h}%KgL`>(iD%#b9vYNUQW>;>TMi+O%nmZpWwe?_%de|#q|GVd+{{iSx_WU~1 zJ?!KRHXAq%+h(dq4M)*_oTl$1Vn%@-I2h-qgY5A-^8^8aC(B8X3`$f91yDLoR=UFt zcgGHL#D~sGW*aUmg{z({jg^VbWc&ZlxKgX?!+A;b7ybuEBrsA{B?o!DgM(tX{7H2M!<)6w$ zB1j62BeA8G*#!7KU3*Cp27ATBgxNirs$=rOWEiA)lV)#3vx^mwm0${v(%b`g{AhB^mWyQ?+mGi^C(YkV10c1vJKG?IJUo_59kh&6DwPH^>jLD8d zX#WJ?L`C#YOdOJpJs7+(68F;`h1Y^*v7j0Jl!2d+A8yM9RUh}|@8=tK0DHJ!OwVKM zI{EX7|Mq68E*&OjCX-3}gvIHmSP%k+)qXRIyij!fk&jY!sQR58+Fb}!QQVGp`4@q< z+6Ir_*{e#%kdkYXQPEvje<^moDnD0++*|6N2{3K6CD~=FGox6lX7qKVpm%&ly5+a8&)M-xf{qb zy~atwwbPDaFSYbiAQn*CFH4jE14b0yY{@y}o`7jz z2S_;8)qRpO@Rbh9J&2AwKqM~k$M4fILu7;Vqf@x4W(ofqNZECr&~7E?9|tX?a_8Dh z!$&P82TBxILCw&}KMA_#J^d9iNwA6CN5M_UBqT*$l#dr63mSi&+jZy7 z@%wdboTtnikFp1+T*PYd{l)%T?FUcrhzl`ihzEf}_wgO(k{y%KSSN z5{(Sz+atDTH@MiDz*om`I`EuP-4X04uf?N_Yzbr8iOlhspXBRR0H!Wk zvH-=9e3RDvayItKLpf5LQfEzq@<&o7Ag`4dt=r_Trg;Zf<65bfS)0Zd&5HZNXlzNS zY8|5XCJq&Hzhd}{lwvP7olGCmVVk@`m%CNM+zkKz%$ zwOck~{WWziy_^g8;x~_3yR#r%62sS=&HOnttU+9a+l`>_QU!+vE2M2^X2nACSSOh1 z2zaSaLC<{MOQFNK8_h34>OXmKQHzEtNO5Pv%=1+)T>^|SqaVN2X_`+bZ@}l+JlYC| zfx}VXeX7oUqvsVX;)HKb1^50vfq0VB6@`+=9vAcBoN?HDu@KK%YJxacVnTIPflPb; zU5RI_j)8)gf{KAzSQGj(4zT5>WFT$G6l>X1ku?|{>3bWv3{9VHbqXn16Zd%)&u`IF zvcvE20564GA@O!2!IIh}vI#Ff!ESZy#IY%ekLKX`#_b#PK(WLPPjg zv<78?QJ$a~7AE`yI2ZC3rEHIaLx`*aC&*tBfFSY;!{yDr?g1{xeo^wZU^v&;1^*GX zv-8Iqa>1_yK_NdF=uDJyll|;hT*0RGY2*j>g$dG8hlV7OZ`%$@w9yu57@*IGAIvmm zvO>4G+taN|q#xSRni}dPW^7nbVHHF4h2c(!svuT>{m7has*}W%E_L5Lk~S^-so*ao zJdQa4wpvz&aFS@HyXva^ktw(STs|3fD@wDpbBx=y=hZc#P89mPhoZEjUoWmHfjEN} zz1B}ERZS*++}0>z#0GsMMj3VfEPiDSbE=g=Gx_D~vNBx`!98r2K4^wKJN03!wa_lB zqQfzuIhKU}_;E!{cwl z&cb3(HYqU?%7NR-QcgMp8!1uHrUwYr$}tA{Gy9;62_mL@k6k<-!4C{h zrD!@UeTJ;i$;DoS_>XUR=yHXG9+LsYHQ8u*xquWIGC7|6u(X|e2Q7@hChKyHJRmi^ zUxT2(O*L~9IV?Ov7l8mOive;p{gyqohT;kNT|b@lLM;IMbdI%UyA-V6EZlXJE~#c8aF)IrH3nOUJfh^92jh9WbltMItdyx3!GYEu73{q4y%N#>(HGyvF^EUtkS+t)U$u zIFZe3wCeSRh>niFs`x@5T2eWEVfhUqu0O}}ST)BL6gAwk@SR;9eW$jM>oO4X#t=L< zCUM$C#0;l8$i`WKA@X!!CZVs`m_7hweabXC;W^OH$R z<*UVBN4=A9`JUSO^dYIOH=GXe*#tV03UsrS<`zol>~$?2>7}3SdbV@J`mX)f@#_J^ zBw$XV9Mz~a?zl{=oxT&~_=C-y&Qne$$IJ{j=WWC`hL)e34}J#k+EdWl;8v|v%rNSn zs2a7I-2;rhl^=JIW}YTUTw_55=0g~XRQ5rlOCx^^({s5eVif(*{8ZLDFPMcKw?*q~ zgElhXVrXXaYs-mI|ISDsj8V~T4aMg7B*oZh%;rk!eZ8kPz9X6vBqk}h9!vw4%#cL5 z{i0nu5KtUQ)Rk_>kzS8#snRWRzOp}2lgv7@X=DIAUwoq@obSe{$y|BwUQH?1O;SJe zM6o-MsxPat7Xi8g9S(p$uH7aqgY~k2?k&f_x;MKMQs6vnf4NUst9I9!uV^}{kjN5-ivw<-=3v08I0 z-=%|+VYnMGhV;b3j7_>EH(>{iFZxi)JynqHD=$4$pPHXfa`izHpN1Q6NWhvB#g_lj z7Ly;uSkM==RDaH4fG-)s69!G`LG-1E)p|jO9?#U1Urb4G>>k!}1{fvlcGx0y4yoB~ zhW)Eqpz_}XVc3e-W#10iT-j+x1Ci*aMGka7GJZzFhgAL0`sD`SnnfszCGm#4lxmCj z;JvdJQku=JvH8d0>V+m}heG1FyWQ zB5e1s-Vk$b&|D+V>8)Y1B9$=~3jeo1*GIcIJp^6;e;1fb0)?PCZm*|zBy!ph-ZN86 zOf!oEd(L4u0-2w8qJf07af?A{CK$0hgfF^-BbQH=D+EePwcWtJ=i6~aT9zC!uN2f1 z5@8)i3fq*B>t%e7;)T5xM>MOuNasnLg;{r4i@2o4ZgGOL_LX+wWDBX{W6np`@&)&8 zhc`Jb@FuuRkDn}DvMR6pzte=e{5_o>9&ZmXA9k-Qdfp#yEFA-0U#>bD4sv6+yWfgS zpoL&@a2F^|jUl1|qZ6U^67l%E)n_cy=o)jCAmmd9*O%3bjX|4a`*pLc5P2No*yc=2 z{)13c%vOTu&H5G!d};QwLfc z@bqQI>8qABo#6MFqHw};{^{2e5A4GSk1W+`YwSJw3V*Vsb0m-uX=J!VvNQgZqrf40 zyYeR3XTb-kQ5yT91#`7%fa~hpEI|~Lm%`J;36USt8?;*ZgXixTrHombD@z85IkXkg zf-+akuPKhTnt|=>m1AFu6>~=$X{_3e9#%V%hjE`4c~fGS-GiFE$~2TTI(XwPg4F4= zd=_|1ebhAQNjmb7igDeG16@6XKlWBu;W-dL*B(VDZXH{2ZF;X8@oj1ydI)WTu4{>H z;*aekhRbRODs1qj)HTYP!!~jr?xCFTeN=Qwi=(&6dYwg4>SW}i)*S};D~hG}BFl2A z&Ey#jt2CnBRsYtm8AwrF90S%=0Au@i90h$hflhQI(}W`?hxn~&tedHa{i$2A2P&&o zp_;12*l=GoxLDY(qwK5|pWC%F5kTy`}~rFYbAz5E_*VkMO+ALeZgQfenC zmh?H&=-v_>3ed@_nyBfe5^OYPBpSOvC)XFt)ftWFs(i&bPiH%u=x~dM0N=G%{qC7O zawVCoL|3s-H}iM059`fW^T*Lm<(@f+XmzK|%{&DvQbp1fB5fY&Xzkso*%gYDz`Y^! zbZmQX6(!VLCR-3Qqcj{R!dvoeRd;n^MBXQd6-t_Q&_-(P>jIvu?mh8=C<9H-w6O8e zpN4OWo;k}OAo|aP_+9Awpe0$Tx|B(RL|*YN3MW7Rjjw=VJ|zSZhF1CAF*D;ih72S!%!#k6%{jonUk(d-m6%+qwIyo_*&kG+=|-6S`qp2%-gTQiPeWETC+KG~UbhwJy%`G+f5jI+kpdtLlVw6-t-#brgj( zkeHL!w2Qiy7t*weW&CZIk=nzXR!eCLN!g>2wxEeCj?+ksMBw*|3f-SIAS*HV*n~xk z-(S8~#UZ8tebu2S(|P;EDlr1>-gqO6&Q)*BedMYQIRAbVDt9$xB7|;c`Lt~z0@khU zYzMXicSde<_InV~dxeY%Kq+Q3@@G;RT7=KcW3emM&%qr;@afL=IuLadaDwt~v-Y<+Fo+yHQq7C03ajfISB6Ej1_cW( z#88@WVi;8Lr~_7oW@CBzTmnLK`Dx?$SB>SDMfP|tuh~4uJ?6sZVm%chvGZnB{Jx~AkzxFX`YeV8KxvI`>%~NqTg)AmE`?ma z9Y~1%RAU_a88J&f-l{;HQ==v*x0~ke>h7H!?BRhB0xXRpC5fh zX-kS-oZ;B#mpIb6C6#B36!6Db>$$u{-^F8?GVhzeLN2DaALq2TdnyLPMj1P)rP3_@ zZ-p?)W@Eq%V#!HKmnI0S`YC$K zPI4SM;o6t((vo$%JZm(vyt0DxI4Judnc3F}$mws-i3Q^DI*IIKR=dzO3Ns27orMPB z5&K$!zTUF?g=Pj$32Fk*%=tIa1A)l`&ozE5-KTfJQK2>U<%OUyV>cegFIYu^Xtfvp zzG#R8ayK$c-{Eh@0=))paAy&#x_&l zw9J5k&H5|AhZf@f!0VG0(h=DQ$`dkGvV^37Ywuy(GVC9lParh0W&q9h=EIF{qFsbs zIY3dgM# ziQmzRREMqE%(FB#Ol7x$(hG2$mLUd;7<(JZe#u7QU2M0V3gPc58i(T85WM4(Z712e zxl-Q4fMiiPAtujbifGe)Jm>_%HUpSoC>dlgH;FAGB+R4%0Fl7WWwHw3Q?HBP5iuY8 z1T{oxSIDwc-4T5`WOP_&pFh(D9Jkfa2_}SpU(`GkjmBY-z<|u4M~KJI-^+x}S zY7h3r$k%nz{g}k8UZHn?KvL>6+YI}+ne@VYW&3)`C!FZ~RnGLvkO zPfag?x3nOeI9(Y~ z8gRGchm4cu0EDz62Rn7ft0%f6;kAn?{`B^NY07!@88q!defqNJ4Y}JY;OZmZD>Gd! ztQe1J(xzqR-Z+}YG#YK7PCp3|}X1)XWO#X$VTyOZAC_8eaF>M^oMp&q|0$ zqM(FBA1DFf1>uugvIUUSy|1$e&J+ZDJE|Ided3_kA`uQ*g5R~>y-Gr{CdJ_E(=#lo zCOV;iW$t!Ej;6S$0=E>J(3YOo0!M#OOi+E)#r<_-lhoGK$H1rgmhB%4&sGWMa@%7_ zI0AK0O0&vOe+a*5KXi`n3NXz~LRa!I6b$rkRSXLWv4mVvE_VujqI;7M33lSO6lay# z%rKEeU@OR)BxAB7`QzM4fGm7@AJ#z?lMSwU6#tyGXs`4pe0d1*XqTZ`exnnmrfkjE{&03P9qZB?KAr)@j0F`t zY%BwW3jOdZ=sS58+vIq56m2GCyevl%jFJc_zyu!}85yaEnyoCVG`yK+Z=a`(2z7b< z?|ItQ69xs>R7X?KAg7gWqBWnckA#?(ZjrEaT0*jmE0wGC5$Aj972mQD;`Ttv`>AZ9 zEkN$9Yh3@!Z?Z3MS}7h^!GYN z+qP{x>DabyJ0070&gA)TZSCyTzL?sXovPV--kdtG&wW>&A3opf`f}vW6`?ZVgTX98 zD^GApdNP&6UUegxwIL)QT426)^>q^t`RdE(y#YV)WO&Gh9pZJZ(pKalv`V?kZFYAJTqZEMM^J3RJ%v3hA#2iV?t-5>zG zX>-4SlOjr(Ai24W<6hQ5ZHX`YhXqetXk-824`?tcY$V73k+>!!xi#85&OL#C|SrRQEzfy)VayJE<_8u*?B5?_6*rU<6Y8us)>7=h^2=dO0b zlm4O+FnJDs59X+^_jaPRs28p}IFyfK` z79sRY@t&UbfZ7q1Y!e!?io++qHB0|MCriF-O{qr87=f-IC)f6!IYi;Sj5?xbqrWkYXghG<%B@&v`oPC9KdT8P{^sQ7X0gBJ2*mcDEBzgJ z2%bj|j|LTDOMzMiG(tsAh%QL}S^*hACNjY{k73+RTVR4m_j4yLe46T0y^M4cmGc%k zTrl{RV>aqTIdq*KC&K=w4Eo)5#aB$<6?ZS@LK{k>;P3Vud3l67kEa3^h75*ZSynk>gGxzDfuK+Q*+bNhYC{v1aqicWL)zS`$tL^$I)3y|-a3Pnvw#xLgQU5*i6I~i zN`IYFLV>ph|DJZ3Qv3B>G<$nuM^wL33b=U@yXf`{$SFd;*DM-P51-G!j5JIgLb%B9UJ59fysBagfOw5O7 zt5O!?N$L$@W~~kgjglv*=|(qh8nhYm!V3`xYJqn4-O`DrYWXr~M_=C=k?1gxh#D31ZT3!8MCtOvS3Ar#NC~1S*Lz|d z+`GcuLCt{^c9{eR$q~^Ci(-mTfFagX2lB{)&){d~L4$t(KoPwrTT$HG$74Sx(Omrc zd1Bu%7E=%QK{cSh?t-S&NlYu}XB012uC8OJtGy?t5QG^`FEO*zDGVL#HMzWn|E zni2f2kv~#`iHgjA&{W!dDpA!#Z`7=c8Z97xiC9pKC*MEgL#*e_YEPxQD1UjhB18(* zzI@wGW<*abA*iX6W~AV0Eh$TFD>WK9U$}4rEekulGf|O0s~L5oWU$?_*6>j`Wh)Dx z_^jRFg72u-CGRcw+?4L5W|5nCXDOJk7r@Ip~xW(nq!nR#;f{c zs!vzz-6S$SOe;EJOJ>&oS|l~*PiT~*mYGLDyJP7xQKed$WBVg#z8@U7jpfHo#Xdhz z*xhTVhagavZ&@G%R#^+xKJ z0kxjEe-Quimu-9CZoj$djAs-nPD$X$QOW&tm2xPrAQk?ObKTK`$!F}z0Jyw3DG@P! zVUnjR2CT|&Tfh|Rtuet;7XE$_93lbbCnqnw;4FaRLWody7J+D2$k*fm;5lym=pgcDm>TVsJqrIs1^I5i{BIhYf@R|CMhk{(Iss3OsPn_l26g1P5yEEzn6@paSR z#8tKw;U)__S#ph7s~-@$cinv#qgj#7f%eJhpR*3pb_hGC3j@;qRQD%(qm9Xm9Ql&7 zr_fu2O_AF;2nWT&57OP8opF|meHBY85#Ust$`M1ZoG5f@z2&+lmsz0A1$=0 z%M+vu@}qzUtZ9AqK-EqQ_hz6{!mUDj6xy&5G zRl?q9)k039iGU-YrqVBf6x3=M8~jMGFXf86iGV?otY0}Bsff8W_u$?zo;~|dahOYs z@_UKP-xIQI)optK=hc0M-h=i@LKj7(Do$#*l>U+>lX(IS3%ON=Wf7W{9qt3Nivqd< zj=!0Xw+7Ewj3?&fl1K~GBqWpT#>0wc<^^J@N4+o|pRXFSjO^26QQwx}`L(wI9#+ZI z9oVy3N?)B4u*EUWOKa{^2y@5|E3zcVjmX=*!Re{CVS_wgJDL@NSSEgji`7+7g$OwB z`=fj;;Dqn5>n4bc{Ql^(e>b=jDefiPh0cPuT^p($2lQYoS6qHhcPM93q-ekQzABr$ zlEcH!as+{>qZKvsMh*~+Mi@&7kyWh!pCs` z2N&V8Im#$c1?uvh5{(~boQQEx(NSa@qkHOOXv$+h#DAh3Hr`)IjIFXP6P7hbc8(jc@SCEIx-J#+FVbTZFHbXhszTcaz}wt^4M!mY1NyaT4qMesJ?SnGlrt5t$|PRvk90GHXUFsD{Y2_-5WJi!@w z;84ia>|18%O69jxJ>^#G>y7puSDLVvf5S?u zUtI*OK#K0-Dn+V)(K!vhUz^V4GJpEc{LF>J{rJv7_~EhG98S`W>>J*cn&<5_wjV8W z3yGicmfr65!zZ^K{P%cqiTS2{s2z7=F3pl0?c1!es?}k`48fOG+XUl&dYfGAEY+K2 zP6`HId~62>asTRP6Ydp3H@tEwB~v(hsE8_6?hPX7g3l&3d&MK@3PktwTJPCf64<1L ztP9Kx^(0B{9kH}h(gbbE^$^EacNCao^pLj`t=jI_`XLwJUFirA1!snoVvh?v7V;C< z_0NNsYk4G*u_4w!<4h(cRum>Gjm<_VKM5`wo9HJygF>|v-YT8DA=|)6+bJ7Xv00ig zH%c#7+x4uW7qwfIsLJ=*DqUYV=m?#_(h>XL+zPHH*U(KRE3>viqf^c-BvUVK#n9ns z>c_hDY!GyyKPz^FP?W5f#W8cnuzoa8sJ={L80|NY!g)S%?(27}M9 zF*%5KGoKi2d#HMr&P;7W7T1pJQgbKbqer!9OQYBpa>kt?0b=j+go{VGyMgg4c6<5y z8cxE{58cwA!zEUsG)*fB)0e~C6k!K}3xs-n!> z{PasLy2adXbQ{B4ngG8ix4K1Ya{Fwd?L{bnPahv+Bdmr+nQyreDP`2O!OJ3 z6W8yYN~MTg{OvVPV5xj`nbt@@&`QkG_njKgzOP_*Qe^4Q8RnuR* z!Ynl3*oZaFi-NPZCJHIr(ynIm2_1_zosC%)M-1UO6%>GTh-JQT&$e~6m1k5bfdy69 zJ&GSZc2AQnU5IWM8KG`Z3`Zu)BDsWG7yi7i5p?2NHeA5%%TNH z;rnEUeq^wjD?8&k!H4S2NBB6Hd{a*CQT{qbOe|bJfsi)iMJ5H+k|NsE=vQJ{ZQTp5 zLS4b4n9bBj%YJY0xL@By(-f$7)7IFvHD=xaN5k!dVZ4*l%+c~jCEa{7yzgKSDtd2; znn8OLb1;M_KR^7oMAcgb)O*?x;VJ$t+V>5ESh4E4!zdDEt5eeC8grJJ^bzrmc^62W2SModa(vVNB5Hx4DJ2cdWXE8-HZNyV-OrvMS#yD5V)N z3n7aL^xW5kY7K<4R|WT7eV86B5;KGnPvkz+_qkgCAWWJiPf@tsxn*kI7orur5gR&e zhTD3A4EzM})H|8IoSWw{$~Fp47u`4x0ws~vb^1%vC28!%~<_~J)*i_Z%25NnFKvS zh_M=mm{k?q(>`B^4=+>S4|jnaio_gb(tD5p*HjW8u+?63J~3+$49z}Nlm4)oSZ5Hd z=P(!MU+H3VDu+j^BMnv!Sr}1mM~p<2D73-%*!AMwBHgi`&&^`$`u1r!*-j(HCS+9p#t@S&bI>PP? zlhtfp8y_^nU!6VFge%|L8X|C{_uZ5Kz#zsQm1d>AA}G)PG7{6aX01tGt;wsP&wKQ* z2s?|Z9qZI#na!ZG^gfA+Z4T$oHlKtx3Zz))vndj0VmWj%)KH3I5wT@&lm@72{u1d) z5$4+n5S90@XYSd%>9sK1P^x~qk0=(AC0j~#kFFW4_VKbK>Jx4g-1z~)d zx=-H)Vg0_Ev8$@D2$y=AGUFV# z6PHmm3x%|rq+6$>#C3weE&)gdyjOjF%k>2-pqAmKYRd{N4@;<8{W}-Qaz|qe%NMPN zvA|e(F=Nf_KCMa0^sQZUFeAB`C6?CgNt38uFZu4a#9&=xR>twqiul<$Y9*#CZ^HYik0e;1^4<20 z`~Bz0ZX($X-gnB=%jV^$C^ww;JBApgbha{frP&PpeAXX?-3{A2jAS_nIF|-Ooc-%w z7Y+sym%H3@7qZto<3~4<1kp(Bw;3m3!o$+?Bw#8Ut{S9xaT8OPoAtel@QjA$opQK1 z6=Wlbz2+|~q4XHsQ_Mews)NqrM7rttIj&i)@(#lLqoWWM_8UZ*Pd7_J3`>#0^UjGR zGx(UVNe?&;C3iWv1IQ-YnjMZYs@webnJ!!O>;sepDIG>NVgNMR#H*zT*o$=Y_XiD~ zaC!>|AqWu_;-0ndyrodL;-%kWrNEu%qllJ4XCsKTYE=pcn76N%kr<>v*6f1H;%W&8 zO8X_IaTDwPBLl~ozAVWAKgRh0_xjEbw1Zs=${pxeTQ)YUao?GopZ`fkG8NTPKm%Kg zR2O*KW9p(}+@o=XZy#j-u&8e3(P~*C0*OO(<@59cb!M`BWWIqA^N8SW*G|ywp}B7} z>t_XzmvE`d7@iyBbnmb$G-e`U(9F+rsu7!k{;mg0)V+=aEc;a~3uzS$yHSW{Uecx$ zzAo^%5Auj&E3!<`TOwT6Z95^WX}~3%HP0S0#LDU#h6|DZP1AZg0Cgxpc847kSN^_O zP)%Wq!_gjsWVz!K(Mqcj{Q);@n`w5~hf7PaZ>syCss|{v`V}E0LO77B5S52d6t3 zO?l=*Ws-8)X&Iv-L+uKM`c*1fkYRzo>@ALct-EX6F2H%VukY}lyHk7-d&dxV#WbLS zZ0aL7aKKddyC=)r%ViXb8al<$m~+#Bi9%|e{JSm+1CHhtY`>d`^4a0L9tV^X?aDj6e6QqlOiG9u9urYId<2vU6f1=1D?1J1-iwcKN03vFTz~%l*)Cxt<}udhwd`5IWI5=P*-H8Yn&t>k<8P^dq+ z4LJ;ji%;~s*#*;@vadyY9Zl3<8EwjZF$UbJrwV)O3c5S zVKb=%cdI?5It=~c_7ECJ>fXP&h|?{Pcqt19ROI{KT==q0tMlIrd|;DQ2&xM@u*%V?Ll76r8}!>h zImo_d6U~y#7fH6Slf~stkx*!}GLOX*L{#8=XEOCFp3;;L+$JJ@2;U?lF=f{nT|JHh z15=P5zBA95iHC5PQxq4NJrM2<7|AX_L!S%`5a`olMWwJLP7;eb8Fg?WAs%qc$3k@< z8}7c(1v~@TdE-W6d5gfXxP2}@`2H>>iK=_5jE=x?!xpFZ3A#wL=Vz7hgC>Q$3fCh< zn4z2-uBDR`D0-00{=QP{Log7r^Jn`oTl4TPfqG1$npZHijZbM)DYsTCZ!`cByp3K$W{`J9>eW02=6UT)00U|7`_bvr7sLeKz|_g42UTJo^TtH`h8^2@bTzGlp=4FU_VzluJKlJyvSX0Qr0Alh(OlV_1VEWE4nNiT3tO9+*DWyN}yvwx==$&lu9ILdxZ<+*3nBbP;1leI`sI zHVoEFzZ0>PXm!8=Mo7|KGBIef2kIs7?I!wC_zQg;l2r;d3gcbzO=7;c{RF2>$=YG>Wnw&;07&vW_x<&b{KY~Gx^dLYW`k*qp4y- z5`(~crh}#Q<^Is99X}?aS(roLqACt?gH2Ec8b~;tcLKrQ!5Hk_HklG-{PbU_plE0s zvgm{g_{V%W{u<3-;!n}zko}CPl?hspq<<|8Q7Y{Sd3^d#(8D8YdX`hGh$0tHXXi*P z?!orKF=(eGAfM<{+9$@aqSJoq*uRbPly}4&wjsnw=z+3J3V`G4o}K}pdPof8 z>8SIYWs#!g&Gt=!2P}2RH3O4^4u)afkgMk*`Dn-!{H%N$fOd~ExvNyU7w*Uk= z8Q?!C57ZG&dWps0^LlV&?1;mFR-hIc6J9mORvwJ%GGEqB;FR_!Jz03$P`78i#pwpK zx6(N#wQr?{rw3Z8O%)HRw0WCsRL$boitZ9(w4t>SCa(Z%&Ld9(uG>~{9O((2M z7^!cooFN#-V+un9h4}n%e$?;Sf$CZqfpuhknSfMILJ2NHBSB#9Ogey2iZVQYXQ?<_ z@b0og?1n2@AcC?PCQ6|i`0DM?Ju78044;qGKb2r?DSzTo=(;i~gcY$nK9|E2hR2$5 zz())9L*iLYjsK|xlfDGUACrm3kc>HQXD2u)uevFy^6kzUZ1hF52I!6sYJXtT& zx!^dVboV6qPbJu(fWUb_GtAt%Gu8=}u*rx*Zh{L-_aE=bR{ULcObODQ2B2si7~y`2 zVMSNYlOK-l2nE6Vy8qn7k&wTZUTu2ETB!j+0WC?CBT@E&msb3i9I&2arW@5HHnh20 z-1zmi1+M?Kk_T=1W6w4;hqES6&8QN1`zMQ7NL)8lHVf4mBBr(x{1jOqIl(m{f3uhx zmfYUFLYrlu=NPKcpFS{+YO0gSscNRWG|eiHh5eTHB-`v+Damwy;4r#~S+{y_s*@Yy zB(#+I?4g#rVJLZfDO1zy$8#H`Nj1B+fJtLa4kcbn_=KbrN_(=ra6L$bM~YYI&G2}8 z%kLbdN^K*p2^MPb=%Cqj6GUVM@6I^X20#1n3#G|_a*%>?bK+PT4zn>)8wbYvf9ZHz z|C57MD8QJ1>@1eAFwr1B^@v@#UQe7t0hZ>$qDat63F4FFr2p*tLxB|QGC-b;`7;Dx zWle;WUL1hXT1hthtCvN=!oWK^${|hschQ@62Lm&(@j&?P7)JRYf=rR{xV4I5mNHUP zCLvvRY~aIQ1RqN$j!Z&555PEN>5*iF z$3)hx7Q_x{pG*Sxh?q=Y(so0lc)!;9bUJL(%!L_xA^uvg=0DNQLg>XMnQ4;}Oc;Jo z0EHALGFhQtA)?_sO`8kQ##v`7KD4uJDl|)N%ul=q`JW?SGyoHV9AO;36VE3>&_}rS zrEO`b)Vx7jVwbb1NP{z;=)_Ra7N-eiOc`q^jGf zC`vl)+RWf|aPS?)!nDwml!H=t9lBD`XxPGt(8QiYLaY@E+u zr_Lx8plC@Pt{Jc7nTwHqk^FE;8UlRMqJCfDd|R2(G93N3i7}L) z;Z}+EVW@FBMt`-;NeIFs_YNx$y9Kdw}bGkTMaYX3o$!KU5|eTBSv`?;TyQj_6>p5#cf^ zf{o)JB+rYln+Vv?KtbOfxY1{$6JCr6-z$xdVlN^5nw;ATV)5ICozR^Y>@pydZz=W( z5o0~u5W2Q=f%7MPeP9!(sstfAh=ggPoEPG2*7t0T20}(H_;1u~Ahaz`|GH*qMxgJZ zCB%>VPCV>=8i-$%u-6=r?d}@c2Jv->f5pt+oF1-VKR|rVZjP8|dh8!$fecasKoZj3VPVDwgq(TJZrzuFBCjRigrDF5qF{Taqw5ZO07QFtv zN#AcQV5G-Z{l|W-X?|~lFX9!VE9aGw1F|FyPLxdQun||RO)hO7oL#PT-|bwovTcj} z**INi`GKz9mjw03J(q;_hT+ZoUPjjla@1^fNTF!Z3D=hLH%E5v#!E%nX^$09Seq4V z1HRY#bT<$s2Katkk=!CLk!`( zT(a>^U&qTJ`*R94W5X%{RU^Rr@qH>&t0w5$UA5&_0y1!p5Hq=*Ot0#cIln~W^KP5v zHzw(exh`e6TTvJZC6Xqz<~cQC9jb;}`&Ui%dNHe7(VLO4ppn1c1?&?)DO3>t=FDYH zw*kThThnzi`{W-K6mGKc<5wKS3>ZWivzFWS!@44~WLkYTpw{UB`Twep%w-LV^oUGW)DNKF+NSF(uOhiLQ<|cVLRacbV~_z^Ms(FkFCoz1iE+@8LI3{tJtVuJ zQpwFwJ6=wq+m}&OUp}h`%4OQQ^Dg$df@pd-rm7TI6lGYN5^yXgA;d{(h*xaxBYlrh z6g;k{&FI&|iMZw5NbwDd0(C*BhM@aB1`Ec%lRBzESOb36RvmPNgoPo}y=bhV8l9bZ zN@y`~9ypI%#IQf2qhb2%Yx44{27bzB*0y>vbqU#Xi#`8I^Dknb4l-mJFgXw0rdCT1 zxoA=Nu;EY!`vyZm-|50ROy+DFD^cRvRG`XLF;dRUU4Id=C!Usp6DBKj3^MJHqbvgv zG(i&;&|OA<7ZRjK{$Nl6<^~lP5oLOc9q8BbW|*NQ?(v7VDwXK)ECfc3@W| zig7$}3R5|=l-xWH+oV8nWOgx%Qb|&ht?nlAp@Vb+3$ba+(kR^+r{=4Ok07j!z(zl? zR-*N@c(j2#+$%IVVKD@f44BV4{D=A6T;WflDfcP2N|pQ3<3tJ@8$Y_BhhVgV?~n7L z{a>cX@FO}xIAYyB?fg;GctS{yhv*f(sX2oVu;OYPX*g=kV;;D(1gOU z7uOg;fe!Ho%)9TG;VrM7_Zl~xSZ;39fdE}yngKKdrF^UfJ@EiN>fH6GRD$AX^_;ld zf6B#)|gMW(H-y*-Q~c zJE|6GTD%f?q{5+tUxi-+zRZMNHdJ4iqYE6Zo1+U@U~W;qYmSHQ&?kX)zk*z55}1(j z1S*+%7>s+SP6K7iHQ=YbZRtWlj_3eo253P3-h>S)-mv~OvSzI!r39^n8$lY&0q`99 zL-MkaKGo+A2D7CIzm0r|tD$uhem#YW4A?GIGH)B7B<(=_5H)86!RU@I5itd|gY49MFCbd&ZUC4tAT(PJ*LJmii=SqPkCJyTh{x@)mY#L#?7F=T08sISCIZvFF@3(eyGPuP8`Q`QhA^s;B5M^3VJa zeTm;AXjh%a0}O;V;ZM^+`Mjzsk%;!XU#p7hG)Gh*(3LNx4P)mv+_I(_335BO;Xij3 zZFg8GXwgvyK6-?L9nQn;Nh|Ywwk#BfK7iE9YX{~nZ& zp2GeQd};QOz-0r*l*&KcPMWISEmvXgpUqxxqmTMOz)L}-eesy5dYc$ilDwW8=2%v+ z*QZ6hYSk-e)LXVhORpUj@sTsWLJVw6ow}zoNt~ndyu>lE9%33Tl{w#hr+&&Ta-NjA z5oMVUOKY4*o$<6B+{VI#NApD965*ld)-$}=86KZie z3dSXKMx%s3e_($zhXPqK_%T1PK3Q-&%6NuqZc1#=evXKD)0{PM1ri@R$?3yPSJN1- zf4M3gILJdN^R5yzFSohf^X!{v?(E2LS(jHIcZM<6V)(g*njs{Qxh@mBO54Rlb=_J$ z$A!0Eh0aeG9HS2_eNFglP)#Pj*l3LYZ zS@M`ly11iqvfBeV^Fu_t-M5rkTomjccGBr1HtUb&(DDDBxW$PnDTg7CK5QJjtG|_O zsQ}uCA!SY&H2d%vOITFC8==5Yb~6H-V%&V?CMluLx?FT4=RAIK^=~i|qlSzwmhHFb z08@ULeB^Vc-8)AZuw^rW01~T_LId_*DE= z4G|QtzM_qXVLjPR$TTr{rRHNPyU~>y@WJ`ycxdz6e6!ok@j%+8a`~FZQw3|Z+tLlM zlC>Y4Q&n*C@@YD&z00$l4K34jsGs4#1G#XkBPi4og>^SxUi=LahSjvXd>NXQZlHqV7$0$b%GjcHDGOithAxWK@=sP zNK=0x!_}T@Wse4l1+^4b5m4C7T#9MS3}0R-SmleQsb>m!di|2T?(-nI8R9jj{%j`y zgp0Vaxd2!W7TUqj(@=Y(x9@#EHyqDoeDw_M-t6=NT=DSoM6EN`_Bd}-)jW;O0VJ`k zy$%Xpe{@u|mfNBd?1Z9#8>X{Rn2Nw~*n@T)kDH`@g7O~0+np%S>sY=^!H6XWPf2Bk zeZ^XtG_LNfk{TZ&rwV!w%+qfXUt1%G5?y?&LOsK-1yRm2#xSL4Pa+PHc>?s-V>?Jp zn{syEdY^f))VMS2fu#FUP;)fXB=ru(GfK%*b!E0Ao!cp9$}Op`3O)OvpI2y({izc( z>kA34wR%FMp*(Y3AsjxQ$)O4l_A(WWF&h+FP#7sSH0z;5Y4vx;A7>Afn2BdOOM` zA!3x5dpCJ{jL7cwbTP&l8uxpI(?oLMFe-$X>~@bydiOU>h?nwv413n09`4!@nvCX# zY#SQxnlXdrL-m`$pC(k@UP)Qh?quCE)8)prY`#;6)vj@@SO2=%Qc;aREyK@Z6mWxU zYwr}lnN(wcZGXdXHA13FJIaf(z($EfrtZe?Qm|#f@+edz1z<*x&FsW?CL|R8wdd;a z-ui(z!&qx{8K-M~ENlqKO&qi4Rys2EL%z$3F18<)Rz7mY_Z=9P;4P|dMm+co^ZFDwL@9C>ql6#N%E?Yz0rL61fY?0NOcA^Nb(LY4dH&MTG1k-OrLlWZRsCMFUUtas}; zqOvLN(7BI!pfpv$l11xBrF8ADT)JIyLCdP=5%S}q?Lvbm)Q-^*XRxDzqjk&3`l2*Q zS}U1xHW3^tcavJleNjQ1d2EZsT@ACjYgbL84O}9NFigp7FR&eu5!*|ws9n|T@X?i; z!f6w*uF{<<9fVWb^0Y(~-?haHYB+;#jA+ZBgJ)CbKKfiH&tb>XzA%ZZu}?-*?K+pN zkV?1XTt1Ix*9Q;BfX`Ujo~-WcrQsGi&%zgf_AYd#uZual$)Q%>+??kWf2ub>p|fzx zU1iM&rTa{AF^j|S3e#@n;vmh5Lfulk47C55>Lu>+|3SS(IBTms{wyWSSQ0r=bKR#* z(MlJ^Y0Kj}1_$`5{^C74-IQ|Rr|U^#YHIHOZUyVi6CV3M=-3RW#J37SVfulEh((kv z%$wQ($6mUpc^RK73~Crx%AR+pj$I<&$AdhIKu|)~DkWqurE50}ev+*<&ND zQu)9l9FaeR8YtAO0!fIwuFpcgV9qC5Y8R1*5kpSz!`@I17{=tc{S5k4pn8%P`kKy; z$P{Y%-7n(Qe$g#47aPl8{zZuH4~H*Pr>`Y=+g-l*rvr1cw*PDGi#hSOCouYY@v%VS zj68qHG6B}h8LoV?u_m(~7PDktkqVopkV}?H+-W{V8+e-@F0&~T2$pURUiR1FRGWAM zzsc$18NZFqIwh>hJrl8PE|2zlH?ouV(oX{0JRG2v?X_JS7xQLaFlX0$8?Ob_Va<+6 zHeQYJM;B4an&dRyl2GNr@?6e})j6gdRti$+>J5r^Z4}EcgI9}=*H{DmNG>HqzH<;& z#3da4ME5SMJZAOfw^#!__asJ{+onO}9yBxwu?4`!mOewf{!6pXWm=cJ$(>T2HY{dr%_>v1Wd}O<6xHyp4p12znSDyz!=EkS z0!f;R#Q*|lXMOFJoq=QB9Y;?9?cLDQTo2{GZ!VYjTjME!o6GI)e?5Om0xXyRSN!Gv ze}KPSBHca3F-Ug2|Maz7D%J%*1<$wrpUht(xv1yyqeTxs&;^6(RA<98YAYyP9HyI6 zx2*5Fx<1I(VlP^y5VS}DaJj78zSy`wL*VbWy}hoUz6x_fsy|_il8dJ)eyTJcLzqu1 zgxy@SzD7xqw}E%4$H&~d>T%*=5OKcBDt9D$z1Dkh5seiJ|NJ!N_z~}*q%83R6)i_K ze3ZC>5zEc`c1dVTUDIv}T&x1B9>iYLm!&{TG}bA4evWdly(qp$3Sov*W|O?Fz~1Np zB)Ro6UfT1;LLmKYNZ_=6EWs2Z+DqasW=+vuHg*@Pp}J~|eWc<#uT{F^DlKa_8DT=Z zUbP4iSt9;y;Q{_M+34*-T{DQz%w7OOM47N{B^vc2$hC0xi&*K$U+04`W`W23FtkdQ z-wx3)A5BBB$bQXP`Q^k^;tu7v3yh=2*1P+=_fx!B6TZEe=6zi1+gmVqcEKoi!5^&H z*v)?dxIEu84Nl)XH+OOEUq-YhP-~G35A~B_9h2NANGmrLZF_HaTRO3@d?K4aE^iSR z=PJKj!SL`;e}A30i2HoDPaYRQZKDfOtTkC+ijrL)T=qw&%q5Ll1v|`l5pq-JUA0R2 zbP0X~uxbrt`*6(tGHmPS9;1i@+4lp)cWis9HL8(H{>BlX1!+BVX0h@Ofz$~u9`9%p zqh`U^}emuJe$UaRfNmhU2Er?@=NH*MU28Mopq zzm8i#-)`O%AeZ;W_xu0-J9%FEVG+wfQmFGZ;&m-4W4#-qeKS4oT+G;qP8!+^QEoJTp1_4GV` z)wq9sVniO#2`3y`-LtD?159NzUU@?Yb{j(PeerT}90Vc*4%OCvx}rxzCo)9jQ&@@w zN_zCQUXPA|I8; z&2ayakX`Ha?8f9Cp7N^8vHARBSRXlZf0G^Q+2hBo|6=9cwDel$0+iUs=l~vub30nEPu=6`1Zlld=z`4_|F!_;^#7j#rsQ7$ s^Dlt;7r^`rVEzR#|6>5=c=!Is{pAD3_sbUu2nYZ$cY0C;N)iS7FBWKY1ONa4 literal 0 HcmV?d00001 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/.editorconfig b/charts/rancher-monitoring/103.2.2+up57.0.3/.editorconfig new file mode 100644 index 0000000000..f5ee2f4610 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[files/dashboards/*.json] +indent_size = 2 +indent_style = space \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/.helmignore new file mode 100644 index 0000000000..9bdbec92b4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/.helmignore @@ -0,0 +1,29 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# helm/charts +OWNERS +hack/ +ci/ +kube-prometheus-*.tgz + +unittests/ +files/dashboards/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/CHANGELOG.md b/charts/rancher-monitoring/103.2.2+up57.0.3/CHANGELOG.md new file mode 100644 index 0000000000..8178169b91 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog +All notable changes from the upstream Prometheus Operator chart will be added to this file. + +## [Package Version 00] - 2020-07-19 +### Added +- Added [Prometheus Adapter](https://github.com/helm/charts/tree/master/stable/prometheus-adapter) as a dependency to the upstream Prometheus Operator chart to allow users to expose custom metrics from the default Prometheus instance deployed by this chart +- Remove `prometheus-operator/cleanup-crds.yaml` and `prometheus-operator/crds.yaml` from the Prometheus Operator upstream chart in favor of just using the CRD directory to install the CRDs. +- Added support for `rkeControllerManager`, `rkeScheduler`, `rkeProxy`, and `rkeEtcd` PushProx exporters for monitoring k8s components within RKE clusters +- Added support for a `k3sServer` PushProx exporter that monitors k3s server components (`kubeControllerManager`, `kubeScheduler`, and `kubeProxy`) within k3s clusters +- Added support for `kubeAdmControllerManager`, `kubeAdmScheduler`, `kubeAdmProxy`, and `kubeAdmEtcd` PushProx exporters for monitoring k8s components within kubeAdm clusters +- Added support for `rke2ControllerManager`, `rke2Scheduler`, `rke2Proxy`, and `rke2Etcd` PushProx exporters for monitoring k8s components within rke2 clusters +- Exposed `prometheus.prometheusSpec.ignoreNamespaceSelectors` on values.yaml and set it to `false` by default. This value instructs the default Prometheus server deployed with this chart to ignore the `namespaceSelector` field within any created ServiceMonitor or PodMonitor CRs that it selects. This prevents ServiceMonitors and PodMonitors from configuring the Prometheus scrape configuration to monitor resources outside the namespace that they are deployed in; if a user needs to have one ServiceMonitor / PodMonitor monitor resources within several namespaces (such as the resources that are used to monitor Istio in a default installation), they should not enable this option since it would require them to create one ServiceMonitor / PodMonitor CR per namespace that they would like to monitor. Relevant fields were also updated in the default README.md. +- Added `grafana.sidecar.dashboards.searchNamespace` to `values.yaml` with a default value of `cattle-dashboards`. The namespace provided should contain all ConfigMaps with the label `grafana_dashboard` and will be searched by the Grafana Dashboards sidecar for updates. The namespace specified is also created along with this deployment. All default dashboard ConfigMaps have been relocated from the deployment namespace to the namespace specified +- Added `monitoring-admin`, `monitoring-edit`, and `monitoring-view` default `ClusterRoles` to allow admins to assign roles to users to interact with Prometheus Operator CRs. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `ClusterRoleBinding` to bind these roles to a Subject to allow them to set up or view `ServiceMonitors` / `PodMonitors` / `PrometheusRules` and view `Prometheus` or `Alertmanager` CRs across the cluster. If `.Values.global.rbac.userRoles.aggregateRolesForRBAC` is enabled, these ClusterRoles will aggregate into the respective default ClusterRoles provided by Kubernetes +- Added `monitoring-config-admin`, `monitoring-config-edit` and `monitoring-config-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `Secrets` and `ConfigMaps` within the `cattle-monitoring-system` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-monitoring-system` namespace to allow them to modify Secrets / ConfigMaps tied to the deployment, such as your Alertmanager Config Secret. +- Added `monitoring-dashboard-admin`, `monitoring-dashboard-edit` and `monitoring-dashboard-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `ConfigMaps` within the `cattle-dashboards` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`) and deploying Grafana as part of this chart. In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-dashboards` namespace to allow them to create / modify ConfigMaps that contain the JSON used to persist Grafana Dashboards on the cluster. +- Added default resource limits for `Prometheus Operator`, `Prometheus`, `AlertManager`, `Grafana`, `kube-state-metrics`, `node-exporter` +- Added a default template `rancher_defaults.tmpl` to AlertManager that Rancher will offer to users in order to help configure the way alerts are rendered on a notifier. Also updated the default template deployed with this chart to reference that template and added an example of a Slack config using this template as a comment in the `values.yaml`. +- Added support for private registries via introducing a new field for `global.cattle.systemDefaultRegistry` that, if supplied, will automatically be prepended onto every image used by the chart. +- Added a default `nginx` proxy container deployed with Grafana whose config is set in the `ConfigMap` located in `charts/grafana/templates/nginx-config.yaml`. The purpose of this container is to make it possible to view Grafana's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8080` (with a `portName` of `nginx-http` instead of the default `service`), which is also where the Grafana service will now point to, and will forward all requests to the Grafana container listening on the default port `3000`. +- Added a default `nginx` proxy container deployed with Prometheus whose config is set in the `ConfigMap` located in `templates/prometheus/nginx-config.yaml`. The purpose of this container is to make it possible to view Prometheus's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8081` (with a `portName` of `nginx-http` instead of the default `web`), which is also where the Prometheus service will now point to, and will forward all requests to the Prometheus container listening on the default port `9090`. +- Added support for passing CIS Scans in a hardened cluster by introducing a Job that patches the default service account within the `cattle-monitoring-system` and `cattle-dashboards` namespaces on install or upgrade and adding a default allow all `NetworkPolicy` to the `cattle-monitoring-system` and `cattle-dashboards` namespaces. +### Modified +- Updated the chart name from `prometheus-operator` to `rancher-monitoring` and added the `io.rancher.certified: rancher` annotation to `Chart.yaml` +- Modified the default `node-exporter` port from `9100` to `9796` +- Modified the default `nameOverride` to `rancher-monitoring`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Modified the default `namespaceOverride` to `cattle-monitoring-system`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Configured some default values for `grafana.service` values and exposed them in the default README.md +- The default namespaces the following ServiceMonitors were changed from the deployment namespace to allow them to continue to monitor metrics when `prometheus.prometheusSpec.ignoreNamespaceSelectors` is enabled: + - `core-dns`: `kube-system` + - `api-server`: `default` + - `kube-controller-manager`: `kube-system` + - `kubelet`: `{{ .Values.kubelet.namespace }}` +- Disabled the following deployments by default (can be enabled if required): + - `AlertManager` + - `kube-controller-manager` metrics exporter + - `kube-etcd` metrics exporter + - `kube-scheduler` metrics exporter + - `kube-proxy` metrics exporter +- Updated default Grafana `deploymentStrategy` to `Recreate` to prevent deployments from being stuck on upgrade if a PV is attached to Grafana +- Modified the default `SelectorNilUsesHelmValues` to default to `false`. As a result, we look for all CRs with any labels in all namespaces by default rather than just the ones tagged with the label `release: rancher-monitoring`. +- Modified the default images used by the `rancher-monitoring` chart to point to Rancher mirrors of the original images from upstream. +- Modified the behavior of the chart to create the Alertmanager Config Secret via a pre-install hook instead of using the normal Helm lifecycle to manage the secret. The benefit of this approach is that all changes to the Config Secret done on a live cluster will never get overridden on a `helm upgrade` since the secret only gets created on a `helm install`. If you would like the secret to be cleaned up on an `helm uninstall`, enable `alertmanager.cleanupOnUninstall`; however, this is disabled by default to prevent the loss of alerting configuration on an uninstall. This secret will never be modified on a `helm upgrade`. +- Modified the default `securityContext` for `Pod` templates across the chart to `{"runAsNonRoot": "true", "runAsUser": "1000"}` and replaced `grafana.rbac.pspUseAppArmor` in favor of `grafana.rbac.pspAnnotations={}` in order to make it possible to deploy this chart on a hardened cluster which does not support Seccomp or AppArmor annotations in PSPs. Users can always choose to specify the annotations they want to use for the PSP directly as part of the values provided. +- Modified `.Values.prometheus.prometheusSpec.containers` to take in a string representing a template that should be rendered by Helm (via `tpl`) instead of allowing a user to provide YAML directly. +- Modified the default Grafana configuration to auto assign users who access Grafana to the Viewer role and enable anonymous access to Grafana dashboards by default. This default works well for a Rancher user who is accessing Grafana via the `kubectl proxy` on the Rancher Dashboard UI since anonymous users who enter via the proxy are authenticated by the k8s API Server, but you can / should modify this behavior if you plan on exposing Grafana in a way that does not require authentication (e.g. as a `NodePort` service). +- Modified the default Grafana configuration to add a default dashboard for Rancher on the Grafana home page. \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/CONTRIBUTING.md b/charts/rancher-monitoring/103.2.2+up57.0.3/CONTRIBUTING.md new file mode 100644 index 0000000000..f6ce2a3235 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contributing Guidelines + +## How to contribute to this chart + +1. Fork this repository, develop and test your Chart. +1. Bump the chart version for every change. +1. Ensure PR title has the prefix `[kube-prometheus-stack]` +1. When making changes to rules or dashboards, see the README.md section on how to sync data from upstream repositories +1. Check the `hack/minikube` folder has scripts to set up minikube and components of this chart that will allow all components to be scraped. You can use this configuration when validating your changes. +1. Check for changes of RBAC rules. +1. Check for changes in CRD specs. +1. PR must pass the linter (`helm lint`) diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/Chart.yaml new file mode 100644 index 0000000000..d71222b02d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/Chart.yaml @@ -0,0 +1,148 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + - name: Upstream Project + url: https://github.com/prometheus-operator/kube-prometheus + artifacthub.io/operator: "true" + catalog.cattle.io/auto-install: rancher-monitoring-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/deploys-on-os: windows + catalog.cattle.io/display-name: Monitoring + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: monitoring.coreos.com.prometheus/v1 + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.19.0-0' + catalog.cattle.io/release-name: rancher-monitoring + catalog.cattle.io/requests-cpu: 4500m + catalog.cattle.io/requests-memory: 4000Mi + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: monitoring + catalog.cattle.io/upstream-version: 57.0.3 +apiVersion: v2 +appVersion: v0.72.0 +dependencies: +- condition: grafana.enabled + name: grafana + repository: file://./charts/grafana + version: 7.3.11 +- condition: hardenedKubelet.enabled + name: hardenedKubelet + repository: file://./charts/hardenedKubelet + version: 0.2.1 +- condition: hardenedNodeExporter.enabled + name: hardenedNodeExporter + repository: file://./charts/hardenedNodeExporter + version: 0.2.1 +- condition: k3sServer.enabled + name: k3sServer + repository: file://./charts/k3sServer + version: 0.2.1 +- condition: kubeStateMetrics.enabled + name: kube-state-metrics + repository: file://./charts/kube-state-metrics + version: 5.16.4 +- condition: kubeAdmControllerManager.enabled + name: kubeAdmControllerManager + repository: file://./charts/kubeAdmControllerManager + version: 0.2.1 +- condition: kubeAdmEtcd.enabled + name: kubeAdmEtcd + repository: file://./charts/kubeAdmEtcd + version: 0.2.1 +- condition: kubeAdmProxy.enabled + name: kubeAdmProxy + repository: file://./charts/kubeAdmProxy + version: 0.2.1 +- condition: kubeAdmScheduler.enabled + name: kubeAdmScheduler + repository: file://./charts/kubeAdmScheduler + version: 0.2.1 +- condition: prometheus-adapter.enabled + name: prometheus-adapter + repository: file://./charts/prometheus-adapter + version: 4.2.0 +- condition: nodeExporter.enabled + name: prometheus-node-exporter + repository: file://./charts/prometheus-node-exporter + version: 4.30.3 +- condition: rke2ControllerManager.enabled + name: rke2ControllerManager + repository: file://./charts/rke2ControllerManager + version: 0.2.1 +- condition: rke2Etcd.enabled + name: rke2Etcd + repository: file://./charts/rke2Etcd + version: 0.2.1 +- condition: rke2IngressNginx.enabled + name: rke2IngressNginx + repository: file://./charts/rke2IngressNginx + version: 0.2.1 +- condition: rke2Proxy.enabled + name: rke2Proxy + repository: file://./charts/rke2Proxy + version: 0.2.1 +- condition: rke2Scheduler.enabled + name: rke2Scheduler + repository: file://./charts/rke2Scheduler + version: 0.2.1 +- condition: rkeControllerManager.enabled + name: rkeControllerManager + repository: file://./charts/rkeControllerManager + version: 0.2.1 +- condition: rkeEtcd.enabled + name: rkeEtcd + repository: file://./charts/rkeEtcd + version: 0.2.1 +- condition: rkeIngressNginx.enabled + name: rkeIngressNginx + repository: file://./charts/rkeIngressNginx + version: 0.2.1 +- condition: rkeProxy.enabled + name: rkeProxy + repository: file://./charts/rkeProxy + version: 0.2.1 +- condition: rkeScheduler.enabled + name: rkeScheduler + repository: file://./charts/rkeScheduler + version: 0.2.1 +- condition: windowsExporter.enabled + name: windowsExporter + repository: file://./charts/windowsExporter + version: 0.3.1 +description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, + and Prometheus rules combined with documentation and scripts to provide easy to + operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus + Operator. +home: https://github.com/prometheus-operator/kube-prometheus +icon: file://assets/logos/rancher-monitoring.png +keywords: +- operator +- prometheus +- kube-prometheus +kubeVersion: '>=1.19.0-0' +maintainers: +- email: andrew@quadcorps.co.uk + name: andrewgkew +- email: gianrubio@gmail.com + name: gianrubio +- email: github.gkarthiks@gmail.com + name: gkarthiks +- email: kube-prometheus-stack@sisti.pt + name: GMartinez-Sisti +- email: github@jkroepke.de + name: jkroepke +- email: scott@r6by.com + name: scottrigby +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: quentin.bisson@gmail.com + name: QuentinBisson +name: rancher-monitoring +sources: +- https://github.com/prometheus-community/helm-charts +- https://github.com/prometheus-operator/kube-prometheus +type: application +version: 103.2.2+up57.0.3 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/README.md new file mode 100644 index 0000000000..9baf58bb16 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/README.md @@ -0,0 +1,1080 @@ +# kube-prometheus-stack + +Installs the [kube-prometheus stack](https://github.com/prometheus-operator/kube-prometheus), a collection of Kubernetes manifests, [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with [Prometheus](https://prometheus.io/) using the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). + +See the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) README for details about components, dashboards, and alerts. + +_Note: This chart was formerly named `prometheus-operator` chart, now renamed to more clearly reflect that it installs the `kube-prometheus` project stack, within which Prometheus Operator is only one component._ + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3+ + +## Get Helm Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Dependencies + +By default this chart installs additional, dependent charts: + +- [prometheus-community/kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) +- [prometheus-community/prometheus-node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) +- [grafana/grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) + +To disable dependencies during installation, see [multiple releases](#multiple-releases) below. + +_See [helm dependency](https://helm.sh/docs/helm/helm_dependency/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +CRDs created by this chart are not removed by default and should be manually cleaned up: + +```console +kubectl delete crd alertmanagerconfigs.monitoring.coreos.com +kubectl delete crd alertmanagers.monitoring.coreos.com +kubectl delete crd podmonitors.monitoring.coreos.com +kubectl delete crd probes.monitoring.coreos.com +kubectl delete crd prometheusagents.monitoring.coreos.com +kubectl delete crd prometheuses.monitoring.coreos.com +kubectl delete crd prometheusrules.monitoring.coreos.com +kubectl delete crd scrapeconfigs.monitoring.coreos.com +kubectl delete crd servicemonitors.monitoring.coreos.com +kubectl delete crd thanosrulers.monitoring.coreos.com +``` + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +With Helm v3, CRDs created by this chart are not updated by default and should be manually updated. +Consult also the [Helm Documentation on CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions). + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. + +### From 56.x to 57.x + +This version upgrades Prometheus-Operator to v0.72.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 55.x to 56.x + +This version upgrades Prometheus-Operator to v0.71.0, Prometheus to 2.49.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 54.x to 55.x + +This version upgrades Prometheus-Operator to v0.70.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 53.x to 54.x + +Grafana Helm Chart has bumped to version 7 + +Please note Grafana Helm Chart [changelog](https://github.com/grafana/helm-charts/tree/main/charts/grafana#to-700). + +### From 52.x to 53.x + +This version upgrades Prometheus-Operator to v0.69.1, Prometheus to 2.47.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 51.x to 52.x + +This includes the ability to select between using existing secrets or create new secret objects for various thanos config. The defaults have not changed but if you were setting: + +- `thanosRuler.thanosRulerSpec.alertmanagersConfig` or +- `thanosRuler.thanosRulerSpec.objectStorageConfig` or +- `thanosRuler.thanosRulerSpec.queryConfig` or +- `prometheus.prometheusSpec.thanos.objectStorageConfig` + +you will have to need to set `existingSecret` or `secret` based on your requirement + +For instance, the `thanosRuler.thanosRulerSpec.alertmanagersConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + secret: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +or the `thanosRuler.thanosRulerSpec.objectStorageConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + existingSecret: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +### From 50.x to 51.x + +This version upgrades Prometheus-Operator to v0.68.0, Prometheus to 2.47.0 and Thanos to v0.32.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 49.x to 50.x + +This version requires Kubernetes 1.19+. + +We do not expect any breaking changes in this version. + +### From 48.x to 49.x + +This version upgrades Prometheus-Operator to v0.67.1, 0, Alertmanager to v0.26.0, Prometheus to 2.46.0 and Thanos to v0.32.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 47.x to 48.x + +This version moved all CRDs into a dedicated sub-chart. No new CRDs are introduced in this version. +See [#3548](https://github.com/prometheus-community/helm-charts/issues/3548) for more context. + +We do not expect any breaking changes in this version. + +### From 46.x to 47.x + +This version upgrades Prometheus-Operator to v0.66.0 with new CRDs (PrometheusAgent and ScrapeConfig). + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 45.x to 46.x + +This version upgrades Prometheus-Operator to v0.65.1 with new CRDs (PrometheusAgent and ScrapeConfig), Prometheus to v2.44.0 and Thanos to v0.31.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 44.x to 45.x + +This version upgrades Prometheus-Operator to v0.63.0, Prometheus to v2.42.0 and Thanos to v0.30.2. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 43.x to 44.x + +This version upgrades Prometheus-Operator to v0.62.0, Prometheus to v2.41.0 and Thanos to v0.30.1. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +If you have explicitly set `prometheusOperator.admissionWebhooks.failurePolicy`, this value is now always used even when `.prometheusOperator.admissionWebhooks.patch.enabled` is `true` (the default). + +The values for `prometheusOperator.image.tag` & `prometheusOperator.prometheusConfigReloader.image.tag` are now empty by default and the Chart.yaml `appVersion` field is used instead. + +### From 42.x to 43.x + +This version upgrades Prometheus-Operator to v0.61.1, Prometheus to v2.40.5 and Thanos to v0.29.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 41.x to 42.x + +This includes the overridability of container registry for all containers at the global level using `global.imageRegistry` or per container image. The defaults have not changed but if you were using a custom image, you will have to override the registry of said custom container image before you upgrade. + +For instance, the prometheus-config-reloader used to be configured as follow: + +```yaml + image: + repository: quay.io/prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +But it now moved to: + +```yaml + image: + registry: quay.io + repository: prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +### From 40.x to 41.x + +This version upgrades Prometheus-Operator to v0.60.1, Prometheus to v2.39.1 and Thanos to v0.28.1. +This version also upgrades the Helm charts of kube-state-metrics to 4.20.2, prometheus-node-exporter to 4.3.0 and Grafana to 6.40.4. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +This version splits kubeScheduler recording and altering rules in separate config values. +Instead of `defaultRules.rules.kubeScheduler` the 2 new variables `defaultRules.rules.kubeSchedulerAlerting` and `defaultRules.rules.kubeSchedulerRecording` are used. + +### From 39.x to 40.x + +This version upgrades Prometheus-Operator to v0.59.1, Prometheus to v2.38.0, kube-state-metrics to v2.6.0 and Thanos to v0.28.0. +This version also upgrades the Helm charts of kube-state-metrics to 4.18.0 and prometheus-node-exporter to 4.2.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +Starting from prometheus-node-exporter version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i kube-prometheus-stack prometheus-community/kube-prometheus-stack +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 38.x to 39.x + +This upgraded prometheus-operator to v0.58.0 and prometheus to v2.37.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 37.x to 38.x + +Reverted one of the default metrics relabelings for cAdvisor added in 36.x, due to it breaking container_network_* and various other statistics. If you do not want this change, you will need to override the `kubelet.cAdvisorMetricRelabelings`. + +### From 36.x to 37.x + +This includes some default metric relabelings for cAdvisor and apiserver metrics to reduce cardinality. If you do not want these defaults, you will need to override the `kubeApiServer.metricRelabelings` and or `kubelet.cAdvisorMetricRelabelings`. + +### From 35.x to 36.x + +This upgraded prometheus-operator to v0.57.0 and prometheus to v2.36.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 34.x to 35.x + +This upgraded prometheus-operator to v0.56.0 and prometheus to v2.35.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 33.x to 34.x + +This upgrades to prometheus-operator to v0.55.0 and prometheus to v2.33.5. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 32.x to 33.x + +This upgrades the prometheus-node-exporter Chart to v3.0.0. Please review the changes to this subchart if you make customizations to hostMountPropagation. + +### From 31.x to 32.x + +This upgrades to prometheus-operator to v0.54.0 and prometheus to v2.33.1. It also changes the default for `grafana.serviceMonitor.enabled` to `true. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 30.x to 31.x + +This version removes the built-in grafana ServiceMonitor and instead relies on the ServiceMonitor of the sub-chart. +`grafana.serviceMonitor.enabled` must be set instead of `grafana.serviceMonitor.selfMonitor` and the old ServiceMonitor may +need to be manually cleaned up after deploying the new release. + +### From 29.x to 30.x + +This version updates kube-state-metrics to 4.3.0 and uses the new option `kube-state-metrics.releaseLabel=true` which adds the "release" label to kube-state-metrics labels, making scraping of the metrics by kube-prometheus-stack work out of the box again, independent of the used kube-prometheus-stack release name. If you already set the "release" label via `kube-state-metrics.customLabels` you might have to remove that and use it via the new option. + +### From 28.x to 29.x + +This version makes scraping port for kube-controller-manager and kube-scheduler dynamic to reflect changes to default serving ports +for those components in Kubernetes versions v1.22 and v1.23 respectively. + +If you deploy on clusters using version v1.22+, kube-controller-manager will be scraped over HTTPS on port 10257. + +If you deploy on clusters running version v1.23+, kube-scheduler will be scraped over HTTPS on port 10259. + +### From 27.x to 28.x + +This version disables PodSecurityPolicies by default because they are deprecated in Kubernetes 1.21 and will be removed in Kubernetes 1.25. + +If you are using PodSecurityPolicies you can enable the previous behaviour by setting `kube-state-metrics.podSecurityPolicy.enabled`, `prometheus-node-exporter.rbac.pspEnabled`, `grafana.rbac.pspEnabled` and `global.rbac.pspEnabled` to `true`. + +### From 26.x to 27.x + +This version splits prometheus-node-exporter chart recording and altering rules in separate config values. +Instead of `defaultRules.rules.node` the 2 new variables `defaultRules.rules.nodeExporterAlerting` and `defaultRules.rules.nodeExporterRecording` are used. + +Also the following defaultRules.rules has been removed as they had no effect: `kubeApiserverError`, `kubePrometheusNodeAlerting`, `kubernetesAbsent`, `time`. + +The ability to set a rubookUrl via `defaultRules.rules.rubookUrl` was reintroduced. + +### From 25.x to 26.x + +This version enables the prometheus-node-exporter subchart servicemonitor by default again, by setting `prometheus-node-exporter.prometheus.monitor.enabled` to `true`. + +### From 24.x to 25.x + +This version upgrade to prometheus-operator v0.53.1. It removes support for setting a runbookUrl, since the upstream format for runbooks changed. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 23.x to 24.x + +The custom `ServiceMonitor` for the _kube-state-metrics_ & _prometheus-node-exporter_ charts have been removed in favour of the built-in sub-chart `ServiceMonitor`; for both sub-charts this means that `ServiceMonitor` customisations happen via the values passed to the chart. If you haven't directly customised this behaviour then there are no changes required to upgrade, but if you have please read the following. + +For _kube-state-metrics_ the `ServiceMonitor` customisation is now set via `kube-state-metrics.prometheus.monitor` and the `kubeStateMetrics.serviceMonitor.selfMonitor.enabled` value has moved to `kube-state-metrics.selfMonitor.enabled`. + +For _prometheus-node-exporter_ the `ServiceMonitor` customisation is now set via `prometheus-node-exporter.prometheus.monitor` and the `nodeExporter.jobLabel` values has moved to `prometheus-node-exporter.prometheus.monitor.jobLabel`. + +### From 22.x to 23.x + +Port names have been renamed for Istio's +[explicit protocol selection](https://istio.io/latest/docs/ops/configuration/traffic-management/protocol-selection/#explicit-protocol-selection). + +| | old value | new value | +|-|-----------|-----------| +| `alertmanager.alertmanagerSpec.portName` | `web` | `http-web` | +| `grafana.service.portName` | `service` | `http-web` | +| `prometheus-node-exporter.service.portName` | `metrics` (hardcoded) | `http-metrics` | +| `prometheus.prometheusSpec.portName` | `web` | `http-web` | + +### From 21.x to 22.x + +Due to the upgrade of the `kube-state-metrics` chart, removal of its deployment/stateful needs to done manually prior to upgrading: + +```console +kubectl delete deployments.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +or if you use autosharding: + +```console +kubectl delete statefulsets.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +### From 20.x to 21.x + +The config reloader values have been refactored. All the values have been moved to the key `prometheusConfigReloader` and the limits and requests can now be set separately. + +### From 19.x to 20.x + +Version 20 upgrades prometheus-operator from 0.50.x to 0.52.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 18.x to 19.x + +`kubeStateMetrics.serviceMonitor.namespaceOverride` was removed. +Please use `kube-state-metrics.namespaceOverride` instead. + +### From 17.x to 18.x + +Version 18 upgrades prometheus-operator from 0.49.x to 0.50.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 16.x to 17.x + +Version 17 upgrades prometheus-operator from 0.48.x to 0.49.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 15.x to 16.x + +Version 16 upgrades kube-state-metrics to v2.0.0. This includes changed command-line arguments and removed metrics, see this [blog post](https://kubernetes.io/blog/2021/04/13/kube-state-metrics-v-2-0/). This version also removes Grafana dashboards that supported Kubernetes 1.14 or earlier. + +### From 14.x to 15.x + +Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 13.x to 14.x + +Version 14 upgrades prometheus-operator from 0.45.x to 0.46.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 12.x to 13.x + +Version 13 upgrades prometheus-operator from 0.44.x to 0.45.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +``` + +### From 11.x to 12.x + +Version 12 upgrades prometheus-operator from 0.43.x to 0.44.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.44/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +``` + +The chart was migrated to support only helm v3 and later. + +### From 10.x to 11.x + +Version 11 upgrades prometheus-operator from 0.42.x to 0.43.x. Starting with 0.43.x an additional `AlertmanagerConfigs` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.43/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +``` + +Version 11 removes the deprecated tlsProxy via ghostunnel in favor of native TLS support the prometheus-operator gained with v0.39.0. + +### From 9.x to 10.x + +Version 10 upgrades prometheus-operator from 0.38.x to 0.42.x. Starting with 0.40.x an additional `Probes` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +``` + +### From 8.x to 9.x + +Version 9 of the helm chart removes the existing `additionalScrapeConfigsExternal` in favour of `additionalScrapeConfigsSecret`. This change lets users specify the secret name and secret key to use for the additional scrape configuration of prometheus. This is useful for users that have prometheus-operator as a subchart and also have a template that creates the additional scrape configuration. + +### From 7.x to 8.x + +Due to new template functions being used in the rules in version 8.x.x of the chart, an upgrade to Prometheus Operator and Prometheus is necessary in order to support them. First, upgrade to the latest version of 7.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version 7.5.0 +``` + +Then upgrade to 8.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version [8.x.x] +``` + +Minimal recommended Prometheus version for this chart release is `2.12.x` + +### From 6.x to 7.x + +Due to a change in grafana subchart, version 7.x.x now requires Helm >= 2.12.0. + +### From 5.x to 6.x + +Due to a change in deployment labels of kube-state-metrics, the upgrade requires `helm upgrade --force` in order to re-create the deployment. If this is not done an error will occur indicating that the deployment cannot be modified: + +```console +invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/name":"kube-state-metrics"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +If this error has already been encountered, a `helm history` command can be used to determine which release has worked, then `helm rollback` to the release, then `helm upgrade --force` to this new one + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-prometheus-stack +``` + +You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. + +### Rancher Monitoring Configuration + +The following table shows values exposed by Rancher Monitoring's additions to the chart: + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `nameOverride` | Provide a name that should be used instead of the chart name when naming all resources deployed by this chart |`"rancher-monitoring"`| +| `namespaceOverride` | Override the deployment namespace | `"cattle-monitoring-system"` | +| `global.rbac.userRoles.create` | Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets | `true` | +| `global.rbac.userRoles.aggregateToDefaultRoles` | Aggregate default user ClusterRoles into default k8s ClusterRoles | `true` | +| `prometheus-adapter.enabled` | Whether to install [prometheus-adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) within the cluster | `true` | +| `prometheus-adapter.prometheus.url` | A URL pointing to the Prometheus deployment within your cluster. The default value is set based on the assumption that you plan to deploy the default Prometheus instance from this chart where `.Values.namespaceOverride=cattle-monitoring-system` and `.Values.nameOverride=rancher-monitoring` | `http://rancher-monitoring-prometheus.cattle-monitoring-system.svc` | +| `prometheus-adapter.prometheus.port` | The port on the Prometheus deployment that Prometheus Adapter can make requests to | `9090` | +| `prometheus.prometheusSpec.ignoreNamespaceSelectors` | Ignore NamespaceSelector settings from the PodMonitor and ServiceMonitor configs. If true, PodMonitors and ServiceMonitors can only discover Pods and Services within the namespace they are deployed into | `false` | + +The following values are enabled for different distributions via [rancher-pushprox](https://github.com/rancher/dev-charts/tree/master/packages/rancher-pushprox). See the rancher-pushprox `README.md` for more information on what all values can be configured for the PushProxy chart. + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `rkeControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in RKE clusters | `false` | +| `rkeScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in RKE clusters | `false` | +| `rkeProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in RKE clusters | `false` | +| `rkeIngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE clusters | `false` | +| `rkeEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in RKE clusters | `false` | +| `rke2IngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE2 clusters | `false` | +| `k3sServer.enabled` | Create a PushProx installation for monitoring k3s-server metrics (accounts for kube-controller-manager, kube-scheduler, and kube-proxy metrics) in k3s clusters | `false` | +| `kubeAdmControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in kubeAdm clusters | `false` | +| `kubeAdmScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in kubeAdm clusters | `false` | +| `kubeAdmProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in kubeAdm clusters | `false` | +| `kubeAdmEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in kubeAdm clusters | `false` | + + +### Multiple releases + +The same chart can be used to run multiple Prometheus instances in the same cluster if required. To achieve this, it is necessary to run only one instance of prometheus-operator and a pair of alertmanager pods for an HA configuration, while all other components need to be disabled. To disable a dependency during installation, set `kubeStateMetrics.enabled`, `nodeExporter.enabled` and `grafana.enabled` to `false`. + +## Work-Arounds for Known Issues + +### Running on private GKE clusters + +When Google configure the control plane for private clusters, they automatically configure VPC peering between your Kubernetes cluster’s network and a separate Google managed project. In order to restrict what Google are able to access within your cluster, the firewall rules configured restrict access to your Kubernetes pods. This means that in order to use the webhook component with a GKE private cluster, you must configure an additional firewall rule to allow the GKE control plane access to your webhook pod. + +You can read more information on how to add firewall rules for the GKE control plane nodes in the [GKE docs](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) + +Alternatively, you can disable the hooks by setting `prometheusOperator.admissionWebhooks.enabled=false`. + +## PrometheusRules Admission Webhooks + +With Prometheus Operator version 0.30+, the core Prometheus Operator pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent malformed rules from being added to the cluster. + +### How the Chart Configures the Hooks + +A validating and mutating webhook configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. + +1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. +2. The prometheus operator pod is configured to use a TLS proxy container, which will load that certificate. +3. Validating and Mutating webhook configurations are created in the cluster, with their failure mode set to Ignore. This allows rules to be created by the same chart at the same time, even though the webhook has not yet been fully set up - it does not have the correct CA field set. +4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations + +### Alternatives + +It should be possible to use [jetstack/cert-manager](https://github.com/jetstack/cert-manager) if a more complete solution is required, but it has not been tested. + +You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `prometheusOperator.admissionWebhooks.certManager.enabled` value to true. + +### Limitations + +Because the operator can only run as a single pod, there is potential for this component failure to cause rule deployment failure. Because this risk is outweighed by the benefit of having validation, the feature is enabled by default. + +## Developing Prometheus Rules and Grafana Dashboards + +This chart Grafana Dashboards and Prometheus Rules are just a copy from [prometheus-operator/prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) and other sources, synced (with alterations) by scripts in [hack](hack) folder. In order to introduce any changes you need to first [add them to the original repository](https://github.com/prometheus-operator/kube-prometheus/blob/main/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md) and then sync there by scripts. + +## Further Information + +For more in-depth documentation of configuration options meanings, please see + +- [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) +- [Prometheus](https://prometheus.io/docs/introduction/overview/) +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana#grafana-helm-chart) + +## prometheus.io/scrape + +The prometheus operator does not support annotation-based discovery of services, using the `PodMonitor` or `ServiceMonitor` CRD in its place as they provide far more configuration options. +For information on how to use PodMonitors/ServiceMonitors, please see the documentation on the `prometheus-operator/prometheus-operator` documentation here: + +- [ServiceMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-servicemonitors) +- [PodMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-podmonitors) +- [Running Exporters](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/running-exporters.md) + +By default, Prometheus discovers PodMonitors and ServiceMonitors within its namespace, that are labeled with the same release tag as the prometheus-operator release. +Sometimes, you may need to discover custom PodMonitors/ServiceMonitors, for example used to scrape data from third-party applications. +An easy way of doing this, without compromising the default PodMonitors/ServiceMonitors discovery, is allowing Prometheus to discover all PodMonitors/ServiceMonitors within its namespace, without applying label filtering. +To do so, you can set `prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues` and `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues` to `false`. + +## Migrating from stable/prometheus-operator chart + +## Zero downtime + +Since `kube-prometheus-stack` is fully compatible with the `stable/prometheus-operator` chart, a migration without downtime can be achieved. +However, the old name prefix needs to be kept. If you want the new name please follow the step by step guide below (with downtime). + +You can override the name to achieve this: + +```console +helm upgrade prometheus-operator prometheus-community/kube-prometheus-stack -n monitoring --reuse-values --set nameOverride=prometheus-operator +``` + +**Note**: It is recommended to run this first with `--dry-run --debug`. + +## Redeploy with new name (downtime) + +If the **prometheus-operator** values are compatible with the new **kube-prometheus-stack** chart, please follow the below steps for migration: + +> The guide presumes that chart is deployed in `monitoring` namespace and the deployments are running there. If in other namespace, please replace the `monitoring` to the deployed namespace. + +1. Patch the PersistenceVolume created/used by the prometheus-operator chart to `Retain` claim policy: + + ```console + kubectl patch pv/ -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + ``` + + **Note:** To execute the above command, the user must have a cluster wide permission. Please refer [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +2. Uninstall the **prometheus-operator** release and delete the existing PersistentVolumeClaim, and verify PV become Released. + + ```console + helm uninstall prometheus-operator -n monitoring + kubectl delete pvc/ -n monitoring + ``` + + Additionally, you have to manually remove the remaining `prometheus-operator-kubelet` service. + + ```console + kubectl delete service/prometheus-operator-kubelet -n kube-system + ``` + + You can choose to remove all your existing CRDs (ServiceMonitors, Podmonitors, etc.) if you want to. + +3. Remove current `spec.claimRef` values to change the PV's status from Released to Available. + + ```console + kubectl patch pv/ --type json -p='[{"op": "remove", "path": "/spec/claimRef"}]' -n monitoring + ``` + +**Note:** To execute the above command, the user must have a cluster wide permission. Please refer to [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +After these steps, proceed to a fresh **kube-prometheus-stack** installation and make sure the current release of **kube-prometheus-stack** matching the `volumeClaimTemplate` values in the `values.yaml`. + +The binding is done via matching a specific amount of storage requested and with certain access modes. + +For example, if you had storage specified as this with **prometheus-operator**: + +```yaml +volumeClaimTemplate: + spec: + storageClassName: gp2 + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi +``` + +You have to specify matching `volumeClaimTemplate` with 50Gi storage and `ReadWriteOnce` access mode. + +Additionally, you should check the current AZ of your legacy installation's PV, and configure the fresh release to use the same AZ as the old one. If the pods are in a different AZ than the PV, the release will fail to bind the existing one, hence creating a new PV. + +This can be achieved either by specifying the labels through `values.yaml`, e.g. setting `prometheus.prometheusSpec.nodeSelector` to: + +```yaml +nodeSelector: + failure-domain.beta.kubernetes.io/zone: east-west-1a +``` + +or passing these values as `--set` overrides during installation. + +The new release should now re-attach your previously released PV with its content. + +## Migrating from coreos/prometheus-operator chart + +The multiple charts have been combined into a single chart that installs prometheus operator, prometheus, alertmanager, grafana as well as the multitude of exporters necessary to monitor a cluster. + +There is no simple and direct migration path between the charts as the changes are extensive and intended to make the chart easier to support. + +The capabilities of the old chart are all available in the new chart, including the ability to run multiple prometheus instances on a single cluster - you will need to disable the parts of the chart you do not wish to deploy. + +You can check out the tickets for this change [here](https://github.com/prometheus-operator/prometheus-operator/issues/592) and [here](https://github.com/helm/charts/pull/6765). + +### High-level overview of Changes + +#### Added dependencies + +The chart has added 3 [dependencies](#dependencies). + +- Node-Exporter, Kube-State-Metrics: These components are loaded as dependencies into the chart, and are relatively simple components +- Grafana: The Grafana chart is more feature-rich than this chart - it contains a sidecar that is able to load data sources and dashboards from configmaps deployed into the same cluster. For more information check out the [documentation for the chart](https://github.com/grafana/helm-charts/blob/main/charts/grafana/README.md) + +#### Kubelet Service + +Because the kubelet service has a new name in the chart, make sure to clean up the old kubelet service in the `kube-system` namespace to prevent counting container metrics twice. + +#### Persistent Volumes + +If you would like to keep the data of the current persistent volumes, it should be possible to attach existing volumes to new PVCs and PVs that are created using the conventions in the new chart. For example, in order to use an existing Azure disk for a helm release called `prometheus-migration` the following resources can be created: + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc-prometheus-migration-prometheus-0 +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: pvc-prometheus-migration-prometheus-0 + diskURI: /subscriptions/f5125d82-2622-4c50-8d25-3f7ba3e9ac4b/resourceGroups/sample-migration-resource-group/providers/Microsoft.Compute/disks/pvc-prometheus-migration-prometheus-0 + fsType: "" + kind: Managed + readOnly: false + capacity: + storage: 1Gi + persistentVolumeReclaimPolicy: Delete + storageClassName: prometheus + volumeMode: Filesystem +``` + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: prometheus + prometheus: prometheus-migration-prometheus + name: prometheus-prometheus-migration-prometheus-db-prometheus-prometheus-migration-prometheus-0 + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: prometheus + volumeMode: Filesystem + volumeName: pvc-prometheus-migration-prometheus-0 +``` + +The PVC will take ownership of the PV and when you create a release using a persistent volume claim template it will use the existing PVCs as they match the naming convention used by the chart. For other cloud providers similar approaches can be used. + +#### KubeProxy + +The metrics bind address of kube-proxy is default to `127.0.0.1:10249` that prometheus instances **cannot** access to. You should expose metrics by changing `metricsBindAddress` field value to `0.0.0.0:10249` if you want to collect them. + +Depending on the cluster, the relevant part `config.conf` will be in ConfigMap `kube-system/kube-proxy` or `kube-system/kube-proxy-config`. For example: + +```console +kubectl -n kube-system edit cm kube-proxy +``` + +```yaml +apiVersion: v1 +data: + config.conf: |- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + # ... + # metricsBindAddress: 127.0.0.1:10249 + metricsBindAddress: 0.0.0.0:10249 + # ... + kubeconfig.conf: |- + # ... +kind: ConfigMap +metadata: + labels: + app: kube-proxy + name: kube-proxy + namespace: kube-system +``` diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/app-README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/app-README.md new file mode 100644 index 0000000000..3920854384 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/app-README.md @@ -0,0 +1,46 @@ +# Rancher Monitoring and Alerting + + This chart is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) chart. The chart deploys [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) and its CRDs along with [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana), [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) and additional charts / Kubernetes manifests to gather metrics. It allows users to monitor their Kubernetes clusters, view metrics in Grafana dashboards, and set up alerts and notifications. + +For more information on how to use the feature, refer to our [docs](https://rancher.com/docs/rancher/v2.x/en/monitoring-alerting/v2.5/). + +The chart installs the following components: + +- [Prometheus Operator](https://github.com/coreos/prometheus-operator) - The operator provides easy monitoring definitions for Kubernetes services, manages [Prometheus](https://prometheus.io/) and [AlertManager](https://prometheus.io/docs/alerting/latest/alertmanager/) instances, and adds default scrape targets for some Kubernetes components. +- [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) - A collection of community-curated Kubernetes manifests, Grafana Dashboards, and PrometheusRules that deploy a default end-to-end cluster monitoring configuration. +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) - Grafana allows a user to create / view dashboards based on the cluster metrics collected by Prometheus. +- [node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) / [kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) / [rancher-pushprox](https://github.com/rancher/charts/tree/dev-v2.7/packages/rancher-monitoring/rancher-pushprox/charts) - These charts monitor various Kubernetes components across different Kubernetes cluster types. +- [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) - The adapter allows a user to expose custom metrics, resource metrics, and external metrics on the default [Prometheus](https://prometheus.io/) instance to the Kubernetes API Server. + +For more information, review the Helm README of this chart. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Upgrading from 100.0.0+up16.6.0 to 100.1.0+up19.0.3 + +### Noticeable changes: +Grafana: +- `sidecar.dashboards.searchNamespace`, `sidecar.datasources.searchNamespace` and `sidecar.notifiers.searchNamespace` support a list of namespaces now. + +Kube-state-metrics +- the type of `collectors` is changed from Dictionary to List. +- `kubeStateMetrics.serviceMonitor.namespaceOverride` was replaced by `kube-state-metrics.namespaceOverride`. + +### Known issues: +- Occasionally, the upgrade fails with errors related to the webhook `prometheusrulemutate.monitoring.coreos.com`. This is a known issue in the upstream, and the workaround is to trigger the upgrade one more time. [32416](https://github.com/rancher/rancher/issues/32416#issuecomment-828881726) diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/.helmignore new file mode 100644 index 0000000000..8cade1318f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.vscode +.project +.idea/ +*.tmproj +OWNERS diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/Chart.yaml new file mode 100644 index 0000000000..ff6bcb26aa --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/Chart.yaml @@ -0,0 +1,39 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/grafana/helm-charts + - name: Upstream Project + url: https://github.com/grafana/grafana + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-grafana +apiVersion: v2 +appVersion: 10.4.1 +description: The leading tool for querying and visualizing time series and metrics. +home: https://grafana.com +icon: https://artifacthub.io/image/b4fed1a7-6c8f-4945-b99d-096efa3e4116 +keywords: +- monitoring +- metric +kubeVersion: '>=1.26.0-0' +maintainers: +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rluckie@cisco.com + name: rtluckie +- email: maor.friedman@redhat.com + name: maorfr +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: mail@torstenwalter.de + name: torstenwalter +name: grafana +sources: +- https://github.com/grafana/grafana +- https://github.com/grafana/helm-charts +type: application +version: 7.3.11 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/README.md new file mode 100644 index 0000000000..0ff07f297d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/README.md @@ -0,0 +1,770 @@ +# Grafana Helm Chart + +* Installs the web dashboarding system [Grafana](http://grafana.org/) + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 4.0.0 (And 3.12.1) + +This version requires Helm >= 2.12.0. + +### To 5.0.0 + +You have to add --force to your helm upgrade command as the labels of the chart have changed. + +### To 6.0.0 + +This version requires Helm >= 3.1.0. + +### To 7.0.0 + +For consistency with other Helm charts, the `global.image.registry` parameter was renamed +to `global.imageRegistry`. If you were not previously setting `global.image.registry`, no action +is required on upgrade. If you were previously setting `global.image.registry`, you will +need to instead set `global.imageRegistry`. + +## Configuration + +| Parameter | Description | Default | +|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------| +| `replicas` | Number of nodes | `1` | +| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` | +| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` | +| `podDisruptionBudget.apiVersion` | Pod disruption apiVersion | `nil` | +| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` | +| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` | +| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`| +| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `image.registry` | Image registry | `docker.io` | +| `image.repository` | Image repository | `grafana/grafana` | +| `image.tag` | Overrides the Grafana image tag whose default is the chart appVersion (`Must be >= 5.0.0`) | `` | +| `image.sha` | Image sha (optional) | `` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets (can be templated) | `[]` | +| `service.enabled` | Enable grafana service | `true` | +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Kubernetes port where service is exposed | `80` | +| `service.portName` | Name of the port on the service | `service` | +| `service.appProtocol` | Adds the appProtocol field to the service | `` | +| `service.targetPort` | Internal service is port | `3000` | +| `service.nodePort` | Kubernetes service nodePort | `nil` | +| `service.annotations` | Service annotations (can be templated) | `{}` | +| `service.labels` | Custom labels | `{}` | +| `service.clusterIP` | internal cluster service IP | `nil` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | +| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | +| `service.externalIPs` | service external IP addresses | `[]` | +| `service.externalTrafficPolicy` | change the default externalTrafficPolicy | `nil` | +| `headlessService` | Create a headless service | `false` | +| `extraExposePorts` | Additional service ports for sidecar containers| `[]` | +| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.annotations` | Ingress annotations (values are templated) | `{}` | +| `ingress.labels` | Custom labels | `{}` | +| `ingress.path` | Ingress accepted path | `/` | +| `ingress.pathType` | Ingress type of path | `Prefix` | +| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.6/guide/ingress/annotations/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| `ingress.ingressClassName` | Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 | `""` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `extraInitContainers` | Init containers to add to the grafana pod | `{}` | +| `extraContainers` | Sidecar containers to add to the grafana pod | `""` | +| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` | +| `extraLabels` | Custom labels for all manifests | `{}` | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` | +| `persistence.size` | Size of persistent volume claim | `10Gi` | +| `persistence.existingClaim` | Use an existing PVC to persist data (can be templated) | `nil` | +| `persistence.storageClassName` | Type of persistent volume claim | `nil` | +| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` | +| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | +| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | +| `persistence.extraPvcLabels` | Extra labels to apply to a PVC. | `{}` | +| `persistence.subPath` | Mount a sub dir of the persistent volume (can be templated) | `nil` | +| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` | +| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` | +| `initChownData.enabled` | If false, don't reset data ownership at startup | true | +| `initChownData.image.registry` | init-chown-data container image registry | `docker.io` | +| `initChownData.image.repository` | init-chown-data container image repository | `busybox` | +| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | +| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` | +| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` | +| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` | +| `schedulerName` | Alternate scheduler name | `nil` | +| `env` | Extra environment variables passed to pods | `{}` | +| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `envFromSecrets` | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envFromConfigMaps` | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret. (passed through [tpl](https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function)) | `{}` | +| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` | +| `extraSecretMounts` | Additional grafana server secret mounts | `[]` | +| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | +| `extraVolumes` | Additional Grafana server volumes | `[]` | +| `automountServiceAccountToken` | Mounted the service account token on the grafana pod. Mandatory, if sidecars are enabled | `true` | +| `createConfigmap` | Enable creating the grafana configmap | `true` | +| `extraConfigmapMounts` | Additional grafana server configMap volume mounts (values are templated) | `[]` | +| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | +| `plugins` | Plugins to be loaded along with Grafana | `[]` | +| `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `alerting` | Configure grafana alerting (passed through tpl) | `{}` | +| `notifiers` | Configure grafana notifiers | `{}` | +| `dashboardProviders` | Configure grafana dashboard providers | `{}` | +| `dashboards` | Dashboards to import | `{}` | +| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` | +| `grafana.ini` | Grafana's primary configuration | `{}` | +| `global.imageRegistry` | Global image pull registry for all images. | `null` | +| `global.imagePullSecrets` | Global image pull secrets (can be templated). Allows either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). | `[]` | +| `ldap.enabled` | Enable LDAP authentication | `false` | +| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` | +| `ldap.config` | Grafana's LDAP configuration | `""` | +| `annotations` | Deployment annotations | `{}` | +| `labels` | Deployment labels | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | `{}` | +| `podPortName` | Name of the grafana port on the pod | `grafana` | +| `lifecycleHooks` | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers) | `{}` | +| `sidecar.image.registry` | Sidecar image registry | `quay.io` | +| `sidecar.image.repository` | Sidecar image repository | `kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `1.26.0` | +| `sidecar.image.sha` | Sidecar image sha (optional) | `""` | +| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | +| `sidecar.resources` | Sidecar resources | `{}` | +| `sidecar.securityContext` | Sidecar securityContext | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` | +| `sidecar.alerts.enabled` | Enables the cluster wide search for alerts and adds/updates/deletes them in grafana |`false` | +| `sidecar.alerts.label` | Label that config maps with alerts should have to be added | `grafana_alert` | +| `sidecar.alerts.labelValue` | Label value that config maps with alerts should have to be added | `""` | +| `sidecar.alerts.searchNamespace` | Namespaces list. If specified, the sidecar will search for alerts config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.alerts.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.alerts.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.alerts.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/alerting/reload"` | +| `sidecar.alerts.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.alerts.initAlerts` | Set to true to deploy the alerts sidecar as an initContainer. This is needed if skipReload is true, to load any alerts defined at startup time. | `false` | +| `sidecar.alerts.extraMounts` | Additional alerts sidecar volume mounts. | `[]` | +| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | +| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | +| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | +| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` | +| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` | +| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` | +| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` | +| `sidecar.dashboards.provider.type` | Provider type | `file` | +| `sidecar.dashboards.provider.foldersFromFilesStructure` | Allow Grafana to replicate dashboard structure from filesystem. | `false` | +| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | +| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `""` | +| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | +| `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | +| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | +| `sidecar.dashboards.searchNamespace` | Namespaces list. If specified, the sidecar will search for dashboards config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` | +| `sidecar.dashboards.reloadURL` | Full url of dashboards configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/dashboards/reload"` | +| `sidecar.dashboards.skipReload` | Enabling this omits defining the REQ_USERNAME, REQ_PASSWORD, REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` | +| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | +| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `""` | +| `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.datasources.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` | +| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.datasources.initDatasources` | Set to true to deploy the datasource sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any datasources defined at startup time. | `false` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | +| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | +| `sidecar.notifiers.labelValue` | Label value that config maps with notifiers should have to be added | `""` | +| `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.notifiers.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.notifiers.reloadURL` | Full url of notifier configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/notifications/reload"` | +| `sidecar.notifiers.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.notifiers.initNotifiers` | Set to true to deploy the notifier sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any notifiers defined at startup time. | `false` | +| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | +| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | +| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | +| `admin.existingSecret` | The name of an existing secret containing the admin credentials (can be templated). | `""` | +| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` | +| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` | +| `serviceAccount.automountServiceAccountToken` | Automount the service account token on all pods where is service account is used | `false` | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `serviceAccount.create` | Create service account | `true` | +| `serviceAccount.labels` | ServiceAccount labels | `{}` | +| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` | +| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` | +| `rbac.create` | Create and use RBAC resources | `true` | +| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` | +| `rbac.useExistingRole` | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` | +| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `false` | +| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `false` | +| `rbac.extraRoleRules` | Additional rules to add to the Role | [] | +| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | +| `command` | Define command to be executed by grafana container at startup | `nil` | +| `args` | Define additional args if command is used | `nil` | +| `testFramework.enabled` | Whether to create test-related resources | `true` | +| `testFramework.image.registry` | `test-framework` image registry. | `docker.io` | +| `testFramework.image.repository` | `test-framework` image repository. | `bats/bats` | +| `testFramework.image.tag` | `test-framework` image tag. | `v1.4.1` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.securityContext` | `test-framework` securityContext | `{}` | +| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | +| `downloadDashboards.envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | +| `downloadDashboardsImage.registry` | Curl docker image registry | `docker.io` | +| `downloadDashboardsImage.repository` | Curl docker image repository | `curlimages/curl` | +| `downloadDashboardsImage.tag` | Curl docker image tag | `7.73.0` | +| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` | +| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` | +| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | +| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | +| `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` | +| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` | +| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | +| `serviceMonitor.relabelings` | RelabelConfigs to apply to samples before scraping. | `[]` | +| `serviceMonitor.metricRelabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | +| `revisionHistoryLimit` | Number of old ReplicaSets to retain | `10` | +| `imageRenderer.enabled` | Enable the image-renderer deployment & service | `false` | +| `imageRenderer.image.registry` | image-renderer Image registry | `docker.io` | +| `imageRenderer.image.repository` | image-renderer Image repository | `grafana/grafana-image-renderer` | +| `imageRenderer.image.tag` | image-renderer Image tag | `latest` | +| `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` | +| `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` | +| `imageRenderer.env` | extra env-vars for image-renderer | `{}` | +| `imageRenderer.envValueFrom` | Environment variables for image-renderer from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` | +| `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | +| `imageRenderer.podAnnotations ` | image-renderer image-renderer pod annotation | `{}` | +| `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | +| `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | +| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | +| `imageRenderer.service.portName` | image-renderer service port name | `http` | +| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` | +| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` | +| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` | +| `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | +| `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | +| `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | +| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | +| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | +| `imageRenderer.resources` | Set resource limits for image-renderer pods | `{}` | +| `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` | +| `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` | +| `imageRenderer.affinity` | Affinity settings for pod assignment | `{}` | +| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | +| `networkPolicy.ingress` | Enable the creation of an ingress network policy | `true` | +| `networkPolicy.egress.enabled` | Enable the creation of an egress network policy | `false` | +| `networkPolicy.egress.ports` | An array of ports to allow for the egress | `[]` | +| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` | + +### Example ingress with path + +With grafana 6.3 and above + +```yaml +grafana.ini: + server: + domain: monitoring.example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true +ingress: + enabled: true + hosts: + - "monitoring.example.com" + path: "/grafana" +``` + +### Example of extraVolumeMounts and extraVolumes + +Configure additional volumes with `extraVolumes` and volume mounts with `extraVolumeMounts`. + +Example for `extraVolumeMounts` and corresponding `extraVolumes`: + +```yaml +extraVolumeMounts: + - name: plugins + mountPath: /var/lib/grafana/plugins + subPath: configs/grafana/plugins + readOnly: false + - name: dashboards + mountPath: /var/lib/grafana/dashboards + hostPath: /usr/shared/grafana/dashboards + readOnly: false + +extraVolumes: + - name: plugins + existingClaim: existing-grafana-claim + - name: dashboards + hostPath: /usr/shared/grafana/dashboards +``` + +Volumes default to `emptyDir`. Set to `persistentVolumeClaim`, +`hostPath`, `csi`, or `configMap` for other types. For a +`persistentVolumeClaim`, specify an existing claim name with +`existingClaim`. + +## Import dashboards + +There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +dashboards: + default: + some-dashboard: + json: | + { + "annotations": + + ... + # Complete json file here + ... + + "title": "Some Dashboard", + "uid": "abcd1234", + "version": 1 + } + custom-dashboard: + # This is a path to a file inside the dashboards directory inside the chart directory + file: dashboards/custom-dashboard.json + prometheus-stats: + # Ref: https://grafana.com/dashboards/2 + gnetId: 2 + revision: 2 + datasource: Prometheus + loki-dashboard-quick-search: + gnetId: 12019 + revision: 2 + datasource: + - name: DS_PROMETHEUS + value: Prometheus + - name: DS_LOKI + value: Loki + local-dashboard: + url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json +``` + +## BASE64 dashboards + +Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit) +A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk. +If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk. + +### Gerrit use case + +Gerrit API for download files has the following schema: where {project-name} and +{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard +the url value is + +## Sidecar for dashboards + +If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with +a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported +dashboards are deleted/updated. + +A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside +one configmap is currently not properly mirrored in grafana. + +Example dashboard config: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-dashboard + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + [...] +``` + +## Sidecar for datasources + +If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the data sources in grafana can be imported. + +Should you aim for reloading datasources in Grafana each time the config is changed, set `sidecar.datasources.skipReload: false` and adjust `sidecar.datasources.reloadURL` to `http://..svc.cluster.local/api/admin/provisioning/datasources/reload`. + +Secrets are recommended over configmaps for this usecase because datasources usually contain private +data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example values to add a postgres datasource as a kubernetes secret: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-datasources + labels: + grafana_datasource: 'true' # default value for: sidecar.datasources.label +stringData: + pg-db.yaml: |- + apiVersion: 1 + datasources: + - name: My pg db datasource + type: postgres + url: my-postgresql-db:5432 + user: db-readonly-user + secureJsonData: + password: 'SUperSEcretPa$$word' + jsonData: + database: my_datase + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 1000 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false + # allow users to edit datasources from the UI. + editable: false +``` + +Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): + +```yaml +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + # name of the datasource. Required + - name: Graphite + # datasource type. Required + type: graphite + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: + # basic auth username + basicAuthUser: + # basic auth password + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false +``` + +## Sidecar for notifiers + +If the parameter `sidecar.notifiers.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.notifiers.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the notification channels in grafana can be imported. The secrets must be created before +`helm install` so that the notifiers init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because alert notification channels usually contain +private data like SMTP usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](https://grafana.com/docs/grafana/latest/administration/provisioning/#alert-notification-channels): + +```yaml +notifiers: + - name: notification-channel-1 + type: slack + uid: notifier1 + # either + org_id: 2 + # or + org_name: Main Org. + is_default: true + send_reminder: true + frequency: 1h + disable_resolve_message: false + # See `Supported Settings` section for settings supporter for each + # alert notification type. + settings: + recipient: 'XXX' + token: 'xoxb' + uploadImage: true + url: https://slack.com + +delete_notifiers: + - name: notification-channel-1 + uid: notifier1 + org_id: 2 + - name: notification-channel-2 + # default org_id: 1 +``` + +## Sidecar for alerting resources + +If the parameter `sidecar.alerts.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster (namespace defined by `sidecar.alerts.searchNamespace`) and filters out the ones with +a label as defined in `sidecar.alerts.label` (default is `grafana_alert`). The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported alerting resources are updated, however, deletions are a little more complicated (see below). + +This sidecar can be used to provision alert rules, contact points, notification policies, notification templates and mute timings as shown in [Grafana Documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/). + +To fetch the alert config which will be provisioned, use the alert provisioning API ([Grafana Documentation](https://grafana.com/docs/grafana/next/developers/http_api/alerting_provisioning/)). +You can use either JSON or YAML format. + +Example config for an alert rule: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-alert + labels: + grafana_alert: "1" +data: + k8s-alert.yml: |- + apiVersion: 1 + groups: + - orgId: 1 + name: k8s-alert + [...] +``` + +To delete provisioned alert rules is a two step process, you need to delete the configmap which defined the alert rule +and then create a configuration which deletes the alert rule. + +Example deletion configuration: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: delete-sample-grafana-alert + namespace: monitoring + labels: + grafana_alert: "1" +data: + delete-k8s-alert.yml: |- + apiVersion: 1 + deleteRules: + - orgId: 1 + uid: 16624780-6564-45dc-825c-8bded4ad92d3 +``` + +## Statically provision alerting resources +If you don't need to change alerting resources (alert rules, contact points, notification policies and notification templates) regularly you could use the `alerting` config option instead of the sidecar option above. +This will grab the alerting config and apply it statically at build time for the helm file. + +There are two methods to statically provision alerting configuration in Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +alerting: + team1-alert-rules.yaml: + file: alerting/team1/rules.yaml + team2-alert-rules.yaml: + file: alerting/team2/rules.yaml + team3-alert-rules.yaml: + file: alerting/team3/rules.yaml + notification-policies.yaml: + file: alerting/shared/notification-policies.yaml + notification-templates.yaml: + file: alerting/shared/notification-templates.yaml + contactpoints.yaml: + apiVersion: 1 + contactPoints: + - orgId: 1 + name: Slack channel + receivers: + - uid: default-receiver + type: slack + settings: + # Webhook URL to be filled in + url: "" + # We need to escape double curly braces for the tpl function. + text: '{{ `{{ template "default.message" . }}` }}' + title: '{{ `{{ template "default.title" . }}` }}' +``` + +The two possibilities for static alerting resource provisioning are: + +* Inlining the file contents as shown for contact points in the above example. +* Importing a file using a relative path starting from the chart root directory as shown for the alert rules in the above example. + +### Important notes on file provisioning + +* The format of the files is defined in the [Grafana documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/) on file provisioning. +* The chart supports importing YAML and JSON files. +* The filename must be unique, otherwise one volume mount will overwrite the other. +* In case of inlining, double curly braces that arise from the Grafana configuration format and are not intended as templates for the chart must be escaped. +* The number of total files under `alerting:` is not limited. Each file will end up as a volume mount in the corresponding provisioning folder of the deployed Grafana instance. +* The file size for each import is limited by what the function `.Files.Get` can handle, which suffices for most cases. + +## How to serve Grafana with a path prefix (/grafana) + +In order to serve Grafana with a prefix (e.g., ), add the following to your values.yaml. + +```yaml +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + + path: /grafana/?(.*) + hosts: + - k8s.example.dev + +grafana.ini: + server: + root_url: http://localhost:3000/grafana # this host can be localhost +``` + +## How to securely reference secrets in grafana.ini + +This example uses Grafana [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. + +In grafana.ini: + +```yaml +grafana.ini: + [auth.generic_oauth] + enabled = true + client_id = $__file{/etc/secrets/auth_generic_oauth/client_id} + client_secret = $__file{/etc/secrets/auth_generic_oauth/client_secret} +``` + +Existing secret, or created along with helm: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: auth-generic-oauth-secret +type: Opaque +stringData: + client_id: + client_secret: +``` + +Include in the `extraSecretMounts` configuration flag: + +```yaml +- extraSecretMounts: + - name: auth-generic-oauth-secret-mount + secretName: auth-generic-oauth-secret + defaultMode: 0440 + mountPath: /etc/secrets/auth_generic_oauth + readOnly: true +``` + +### extraSecretMounts using a Container Storage Interface (CSI) provider + +This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) + +```yaml +- extraSecretMounts: + - name: secrets-store-inline + mountPath: /run/secrets + readOnly: true + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "my-provider" + nodePublishSecretRef: + name: akv-creds +``` + +## Image Renderer Plug-In + +This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/README.md#run-in-docker) + +```yaml +imageRenderer: + enabled: true +``` + +### Image Renderer NetworkPolicy + +By default the image-renderer pods will have a network policy which only allows ingress traffic from the created grafana instance + +### High Availability for unified alerting + +If you want to run Grafana in a high availability cluster you need to enable +the headless service by setting `headlessService: true` in your `values.yaml` +file. + +As next step you have to setup the `grafana.ini` in your `values.yaml` in a way +that it will make use of the headless service to obtain all the IPs of the +cluster. You should replace ``{{ Name }}`` with the name of your helm deployment. + +```yaml +grafana.ini: + ... + unified_alerting: + enabled: true + ha_peers: {{ Name }}-headless:9094 + ha_listen_address: ${POD_IP}:9094 + ha_advertise_address: ${POD_IP}:9094 + + alerting: + enabled: false +``` diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/dashboards/custom-dashboard.json b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/dashboards/custom-dashboard.json new file mode 100644 index 0000000000..9e26dfeeb6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/dashboards/custom-dashboard.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/NOTES.txt new file mode 100644 index 0000000000..d86419fe23 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/NOTES.txt @@ -0,0 +1,55 @@ +1. Get your '{{ .Values.adminUser }}' user password by running: + + kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} -o jsonpath="{.data.{{ .Values.admin.passwordKey | default "admin-password" }}}" | base64 --decode ; echo + + +2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}.svc.cluster.local +{{ if .Values.ingress.enabled }} + If you bind grafana to 80, please update values in values.yaml and reinstall: + ``` + securityContext: + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + + command: + - "setcap" + - "'cap_net_bind_service=+ep'" + - "/usr/sbin/grafana-server &&" + - "sh" + - "/run.sh" + ``` + Details refer to https://grafana.com/docs/installation/configuration/#http-port. + Or grafana would always crash. + + From outside the cluster, the server URL(s) are: + {{- range .Values.ingress.hosts }} + http://{{ . }} + {{- end }} +{{- else }} + Get the Grafana URL to visit by running these commands in the same shell: + {{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "grafana.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT + {{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc --namespace {{ include "grafana.namespace" . }} -w {{ include "grafana.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ include "grafana.namespace" . }} {{ include "grafana.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + http://$SERVICE_IP:{{ .Values.service.port -}} + {{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ include "grafana.namespace" . }} -l "app.kubernetes.io/name={{ include "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ include "grafana.namespace" . }} port-forward $POD_NAME 3000 + {{- end }} +{{- end }} + +3. Login with the password from step 1 and the username: {{ .Values.adminUser }} + +{{- if not .Values.persistence.enabled }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Grafana pod is terminated. ##### +################################################################################# +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_config.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_config.tpl new file mode 100644 index 0000000000..19df19cd2a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_config.tpl @@ -0,0 +1,171 @@ +{{/* + Generate config map data + */}} +{{- define "grafana.configData" -}} +{{ include "grafana.assertNoLeakedSecrets" . }} +{{- $files := .Files }} +{{- $root := . -}} +{{- with .Values.plugins }} +plugins: {{ join "," . }} +{{- end }} +grafana.ini: | +{{- range $elem, $elemVal := index .Values "grafana.ini" }} + {{- if not (kindIs "map" $elemVal) }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} +{{- end }} +{{- range $key, $value := index .Values "grafana.ini" }} + {{- if kindIs "map" $value }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- range $key, $value := .Values.datasources }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.notifiers }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.alerting }} +{{- if (hasKey $value "file") }} +{{ $key }}: +{{- toYaml ( $files.Get $value.file ) | nindent 2 }} +{{- else if (or (hasKey $value "secret") (hasKey $value "secretFile"))}} +{{/* will be stored inside secret generated by "configSecret.yaml"*/}} +{{- else }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.dashboardProviders }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} + +{{- if .Values.dashboards }} +download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} +{{ $dashboardProviders := .Values.dashboardProviders }} +{{- range $provider, $dashboards := .Values.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -skf \ + --connect-timeout 60 \ + --max-time 60 \ + {{- if not $value.b64content }} + {{- if not $value.acceptHeader }} + -H "Accept: application/json" \ + {{- else }} + -H "Accept: {{ $value.acceptHeader }}" \ + {{- end }} + {{- if $value.token }} + -H "Authorization: token {{ $value.token }}" \ + {{- end }} + {{- if $value.bearerToken }} + -H "Authorization: Bearer {{ $value.bearerToken }}" \ + {{- end }} + {{- if $value.basic }} + -H "Authorization: Basic {{ $value.basic }}" \ + {{- end }} + {{- if $value.gitlabToken }} + -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \ + {{- end }} + -H "Content-Type: application/json;charset=UTF-8" \ + {{- end }} + {{- $dpPath := "" -}} + {{- range $kd := (index $dashboardProviders "dashboardproviders.yaml").providers }} + {{- if eq $kd.name $provider }} + {{- $dpPath = $kd.options.path }} + {{- end }} + {{- end }} + {{- if $value.url }} + "{{ $value.url }}" \ + {{- else }} + "https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download" \ + {{- end }} + {{- if $value.datasource }} + {{- if kindIs "string" $value.datasource }} + | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g' \ + {{- end }} + {{- if kindIs "slice" $value.datasource }} + {{- range $value.datasource }} + | sed '/-- .* --/! s/${{"{"}}{{ .name }}}/{{ .value }}/g' \ + {{- end }} + {{- end }} + {{- end }} + {{- if $value.b64content }} + | base64 -d \ + {{- end }} + > "{{- if $dpPath -}}{{ $dpPath }}{{- else -}}/var/lib/grafana/dashboards/{{ $provider }}{{- end -}}/{{ $key }}.json" + {{ end }} + {{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Generate dashboard json config map data + */}} +{{- define "grafana.configDashboardProviderData" -}} +provider.yaml: |- + apiVersion: 1 + providers: + - name: '{{ .Values.sidecar.dashboards.provider.name }}' + orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + {{- end }} + type: {{ .Values.sidecar.dashboards.provider.type }} + disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} + allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }} + options: + foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} +{{- end -}} + +{{- define "grafana.secretsData" -}} +{{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} +admin-user: {{ .Values.adminUser | b64enc | quote }} +{{- if .Values.adminPassword }} +admin-password: {{ .Values.adminPassword | b64enc | quote }} +{{- else }} +admin-password: {{ include "grafana.password" . }} +{{- end }} +{{- end }} +{{- if not .Values.ldap.existingSecret }} +ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_helpers.tpl new file mode 100644 index 0000000000..68d2d815d8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,305 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "grafana.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "grafana.serviceAccountNameTest" -}} +{{- if .Values.serviceAccount.create }} +{{- default (print (include "grafana.fullname" .) "-test") .Values.serviceAccount.nameTest }} +{{- else }} +{{- default "default" .Values.serviceAccount.nameTest }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "grafana.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.extraLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "grafana.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.imageRenderer.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.imageRenderer.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels ImageRenderer +*/}} +{{- define "grafana.imageRenderer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Looks if there's an existing secret and reuse its password. If not it generates +new password and use it. +*/}} +{{- define "grafana.password" -}} +{{- $secret := (lookup "v1" "Secret" (include "grafana.namespace" .) (include "grafana.fullname" .) ) }} +{{- if $secret }} +{{- index $secret "data" "admin-password" }} +{{- else }} +{{- (randAlphaNum 40) | b64enc | quote }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for rbac. +*/}} +{{- define "grafana.rbac.apiVersion" -}} +{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} +{{- print "rbac.authorization.k8s.io/v1" }} +{{- else }} +{{- print "rbac.authorization.k8s.io/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "grafana.ingress.apiVersion" -}} +{{- if and ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) }} +{{- print "networking.k8s.io/v1" }} +{{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +{{- print "networking.k8s.io/v1beta1" }} +{{- else }} +{{- print "extensions/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "grafana.hpa.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "autoscaling/v2" }} +{{- print "autoscaling/v2" }} +{{- else }} +{{- print "autoscaling/v2beta2" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "grafana.podDisruptionBudget.apiVersion" -}} +{{- if $.Values.podDisruptionBudget.apiVersion }} +{{- print $.Values.podDisruptionBudget.apiVersion }} +{{- else if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" }} +{{- print "policy/v1" }} +{{- else }} +{{- print "policy/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return if ingress is stable. +*/}} +{{- define "grafana.ingress.isStable" -}} +{{- eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1" }} +{{- end }} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "grafana.ingress.supportsIngressClassName" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "grafana.ingress.supportsPathType" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "root" . "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "grafana.imagePullSecrets" -}} +{{- $root := .root }} +{{- range (concat .root.Values.global.imagePullSecrets .imagePullSecrets) }} +{{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml (dict "name" (tpl .name $root)) | trim }} +{{- else }} +- name: {{ tpl . $root }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* + Checks whether or not the configSecret secret has to be created + */}} +{{- define "grafana.shouldCreateConfigSecret" -}} +{{- $secretFound := false -}} +{{- range $key, $value := .Values.datasources }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} + {{- if (or (hasKey $value "secret") (hasKey $value "secretFile")) }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- $secretFound}} +{{- end -}} + +{{/* + Checks whether the user is attempting to store secrets in plaintext + in the grafana.ini configmap +*/}} +{{/* grafana.assertNoLeakedSecrets checks for sensitive keys in values */}} +{{- define "grafana.assertNoLeakedSecrets" -}} + {{- $sensitiveKeysYaml := ` +sensitiveKeys: +- path: ["database", "password"] +- path: ["smtp", "password"] +- path: ["security", "secret_key"] +- path: ["security", "admin_password"] +- path: ["auth.basic", "password"] +- path: ["auth.ldap", "bind_password"] +- path: ["auth.google", "client_secret"] +- path: ["auth.github", "client_secret"] +- path: ["auth.gitlab", "client_secret"] +- path: ["auth.generic_oauth", "client_secret"] +- path: ["auth.okta", "client_secret"] +- path: ["auth.azuread", "client_secret"] +- path: ["auth.grafana_com", "client_secret"] +- path: ["auth.grafananet", "client_secret"] +- path: ["azure", "user_identity_client_secret"] +- path: ["unified_alerting", "ha_redis_password"] +- path: ["metrics", "basic_auth_password"] +- path: ["external_image_storage.s3", "secret_key"] +- path: ["external_image_storage.webdav", "password"] +- path: ["external_image_storage.azure_blob", "account_key"] +` | fromYaml -}} + {{- if $.Values.assertNoLeakedSecrets -}} + {{- $grafanaIni := index .Values "grafana.ini" -}} + {{- range $_, $secret := $sensitiveKeysYaml.sensitiveKeys -}} + {{- $currentMap := $grafanaIni -}} + {{- $shouldContinue := true -}} + {{- range $index, $elem := $secret.path -}} + {{- if and $shouldContinue (hasKey $currentMap $elem) -}} + {{- if eq (len $secret.path) (add1 $index) -}} + {{- if not (regexMatch "\\$(?:__(?:env|file|vault))?{[^}]+}" (index $currentMap $elem)) -}} + {{- fail (printf "Sensitive key '%s' should not be defined explicitly in values. Use variable expansion instead. You can disable this client-side validation by changing the value of assertNoLeakedSecrets." (join "." $secret.path)) -}} + {{- end -}} + {{- else -}} + {{- $currentMap = index $currentMap $elem -}} + {{- end -}} + {{- else -}} + {{- $shouldContinue = false -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_pod.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_pod.tpl new file mode 100644 index 0000000000..2ebf7d5f10 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_pod.tpl @@ -0,0 +1,1296 @@ +{{- define "grafana.pod" -}} +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- $root := . -}} +{{- with .Values.schedulerName }} +schedulerName: "{{ . }}" +{{- end }} +serviceAccountName: {{ include "grafana.serviceAccountName" . }} +automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} +{{- with .Values.securityContext }} +securityContext: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.hostAliases }} +hostAliases: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .Values.dnsPolicy }} +dnsPolicy: {{ .Values.dnsPolicy }} +{{- end }} +{{- with .Values.dnsConfig }} +dnsConfig: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.priorityClassName }} +priorityClassName: {{ . }} +{{- end }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.extraInitContainers (and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts) (and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources) (and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers)) }} +initContainers: +{{- end }} +{{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} + - name: init-chown-data + {{- $registry := include "system_default_registry" . | default .Values.initChownData.image.registry -}} + {{- if .Values.initChownData.image.sha }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}@sha256:{{ .Values.initChownData.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.initChownData.image.pullPolicy }} + {{- with .Values.initChownData.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + command: + - chown + - -R + - {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }} + - /var/lib/grafana + {{- with .Values.initChownData.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} +{{- end }} +{{- if .Values.dashboards }} + - name: download-dashboards + {{- $registry := include "system_default_registry" . | default .Values.downloadDashboardsImage.registry -}} + {{- if .Values.downloadDashboardsImage.sha }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}@sha256:{{ .Values.downloadDashboardsImage.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} + command: ["/bin/sh"] + args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh -x /etc/grafana/download_dashboards.sh" ] + {{- with .Values.downloadDashboards.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + env: + {{- range $key, $value := .Values.downloadDashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.downloadDashboards.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- with .Values.downloadDashboards.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.downloadDashboards.envFromSecret }} + envFrom: + - secretRef: + name: {{ tpl . $root }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts }} + - name: {{ include "grafana.name" . }}-init-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources }} + - name: {{ include "grafana.name" . }}-init-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.datasources.searchNamespace | join ",") . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end }} +{{- if and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers }} + - name: {{ include "grafana.name" . }}-init-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- with .Values.extraInitContainers }} + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 2 }} +{{- end }} +{{- if not .Values.enableKubeBackwardCompatibility }} +enableServiceLinks: {{ .Values.enableServiceLinks }} +{{- end }} +containers: +{{- if and .Values.sidecar.alerts.enabled (not .Values.sidecar.alerts.initAlerts) }} + - name: {{ include "grafana.name" . }}-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.alerts.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.alerts.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.alerts.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.alerts.watchServerTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchServerTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.alerts.watchClientTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchClientTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if .Values.sidecar.dashboards.enabled }} + - name: {{ include "grafana.name" . }}-sc-dashboard + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.dashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.datasources.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.dashboards.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.dashboards.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.dashboards.label }}" + {{- with .Values.sidecar.dashboards.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.dashboards.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.folderAnnotation }} + - name: FOLDER_ANNOTATION + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- if not .Values.sidecar.dashboards.skipReload }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + - name: REQ_URL + value: {{ .Values.sidecar.dashboards.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.dashboards.watchServerTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchServerTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchClientTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchClientTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: {{ .Values.sidecar.dashboards.watchClientTimeout | quote }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- with .Values.sidecar.dashboards.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if and .Values.sidecar.datasources.enabled (not .Values.sidecar.datasources.initDatasources) }} + - name: {{ include "grafana.name" . }}-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.datasources.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.datasources.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.datasources.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.datasources.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.datasources.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.datasources.watchServerTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchServerTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.watchClientTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchClientTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: {{ include "grafana.name" . }}-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.notifiers.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.notifiers.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.notifiers.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.notifiers.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.notifiers.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.notifiers.watchServerTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchServerTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.notifiers.watchClientTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchClientTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: {{ include "grafana.name" . }}-sc-plugins + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.plugins.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.plugins.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.plugins.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.plugins.label }}" + {{- if .Values.sidecar.plugins.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.plugins.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/plugins" + - name: RESOURCE + value: {{ quote .Values.sidecar.plugins.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.plugins.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.plugins.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.plugins.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.plugins.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.plugins.watchServerTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchServerTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.watchClientTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchClientTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" +{{- end}} + - name: {{ .Chart.Name }} + {{- $registry := include "system_default_registry" . | default .Values.image.registry -}} + {{- if .Values.image.sha }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .Values.args }} + args: + {{- range .Values.args }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + {{- if .Values.ldap.enabled }} + - name: ldap + mountPath: "/etc/grafana/ldap.toml" + subPath: ldap.toml + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ tpl (.subPath | default "") $root }} + readOnly: {{ .readOnly }} + {{- end }} + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- with .Values.dashboards }} + {{- range $provider, $dashboards := . }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "json") (hasKey $value "file")) }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardsConfigMaps }} + {{- range (keys . | sortAlpha) }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" + {{- end }} + {{- end }} + {{- with .Values.datasources }} + {{- $datasources := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $datasources .) "secret")) }} {{/*check if current datasource should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.notifiers }} + {{- $notifiers := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $notifiers .) "secret")) }} {{/*check if current notifier should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.alerting }} + {{- $alertingmap := .}} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $.Values.alerting .) "secret") (hasKey (index $.Values.alerting .) "secretFile")) }} {{/*check if current alerting entry should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardProviders }} + {{- range (keys . | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- end}} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml + {{- end}} + {{- end}} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" + {{- end}} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" + {{- end}} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" + {{- end}} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + ports: + - name: {{ .Values.podPortName }} + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + - name: {{ .Values.gossipPortName }}-tcp + containerPort: 9094 + protocol: TCP + - name: {{ .Values.gossipPortName }}-udp + containerPort: 9094 + protocol: UDP + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if .Values.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ include "grafana.fullname" . }} + key: plugins + {{- end }} + {{- if .Values.smtp.existingSecret }} + - name: GF_SMTP_USER + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.userKey | default "user" }} + - name: GF_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.passwordKey | default "password" }} + {{- end }} + {{- if .Values.imageRenderer.enabled }} + - name: GF_RENDERING_SERVER_URL + value: http://{{ include "grafana.fullname" . }}-image-renderer.{{ include "grafana.namespace" . }}:{{ .Values.imageRenderer.service.port }}/render + - name: GF_RENDERING_CALLBACK_URL + value: {{ .Values.imageRenderer.grafanaProtocol }}://{{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} + {{- end }} + - name: GF_PATHS_DATA + value: {{ (get .Values "grafana.ini").paths.data }} + - name: GF_PATHS_LOGS + value: {{ (get .Values "grafana.ini").paths.logs }} + - name: GF_PATHS_PLUGINS + value: {{ (get .Values "grafana.ini").paths.plugins }} + - name: GF_PATHS_PROVISIONING + value: {{ (get .Values "grafana.ini").paths.provisioning }} + {{- range $key, $value := .Values.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- range $key, $value := .Values.env }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" + {{- end }} + {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) .Values.envFromConfigMaps }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ tpl .Values.envFromSecret . }} + {{- end }} + {{- if .Values.envRenderSecret }} + - secretRef: + name: {{ include "grafana.fullname" . }}-env + {{- end }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- range .Values.envFromConfigMaps }} + - configMapRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.lifecycleHooks }} + lifecycle: + {{- tpl (toYaml .) $root | nindent 6 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- with .Values.extraContainers }} + {{- tpl . $ | nindent 2 }} +{{- end }} +nodeSelector: {{ include "linux-node-selector" . | nindent 2 }} +{{- with .Values.nodeSelector }} + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.affinity }} +affinity: + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- with .Values.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} +tolerations: {{ include "linux-node-tolerations" . | nindent 2 }} +{{- with .Values.tolerations }} + {{- toYaml . | nindent 2 }} +{{- end }} +volumes: + - name: config + configMap: + name: {{ include "grafana.fullname" . }} + {{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} + {{- if and .Values.createConfigmap $createConfigSecret }} + - name: config-secret + secret: + secretName: {{ include "grafana.fullname" . }}-config-secret + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.dashboards }} + {{- range (keys .Values.dashboards | sortAlpha) }} + - name: dashboards-{{ . }} + configMap: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.dashboardsConfigMaps }} + {{- range $provider, $name := .Values.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ tpl $name $root }} + {{- end }} + {{- end }} + {{- if .Values.ldap.enabled }} + - name: ldap + secret: + {{- if .Values.ldap.existingSecret }} + secretName: {{ .Values.ldap.existingSecret }} + {{- else }} + secretName: {{ include "grafana.fullname" . }} + {{- end }} + items: + - key: ldap-toml + path: ldap.toml + {{- end }} + {{- if and .Values.persistence.enabled (eq .Values.persistence.type "pvc") }} + - name: storage + persistentVolumeClaim: + claimName: {{ tpl (.Values.persistence.existingClaim | default (include "grafana.fullname" .)) . }} + {{- else if and .Values.persistence.enabled (has .Values.persistence.type $sts) }} + {{/* nothing */}} + {{- else }} + - name: storage + {{- if .Values.persistence.inMemory.enabled }} + emptyDir: + medium: Memory + {{- with .Values.persistence.inMemory.sizeLimit }} + sizeLimit: {{ . }} + {{- end }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + emptyDir: + {{- with .Values.sidecar.alerts.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + emptyDir: + {{- with .Values.sidecar.dashboards.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + configMap: + name: {{ include "grafana.fullname" . }}-config-dashboards + {{- end }} + {{- end }} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + emptyDir: + {{- with .Values.sidecar.datasources.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + emptyDir: + {{- with .Values.sidecar.plugins.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + emptyDir: + {{- with .Values.sidecar.notifiers.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- range .Values.extraSecretMounts }} + {{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else if .projected }} + - name: {{ .name }} + projected: + {{- toYaml .projected | nindent 6 }} + {{- else if .csi }} + - name: {{ .name }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- end }} + {{- end }} + {{- range .Values.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 6 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 6 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 6 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + emptyDir: {} + {{- end }} + {{- with .Values.extraContainerVolumes }} + {{- tpl (toYaml .) $root | nindent 2 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrole.yaml new file mode 100644 index 0000000000..3af4b62b63 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) (not .Values.rbac.useExistingClusterRole) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-clusterrole +{{- if or .Values.sidecar.dashboards.enabled .Values.rbac.extraClusterRoleRules .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} +rules: + {{- if or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end}} + {{- with .Values.rbac.extraClusterRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end}} +{{- end}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..bda9431a2c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "grafana.fullname" . }}-clusterrolebinding + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +roleRef: + kind: ClusterRole + {{- if .Values.rbac.useExistingClusterRole }} + name: {{ .Values.rbac.useExistingClusterRole }} + {{- else }} + name: {{ include "grafana.fullname" . }}-clusterrole + {{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configSecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configSecret.yaml new file mode 100644 index 0000000000..55574b9bbc --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configSecret.yaml @@ -0,0 +1,43 @@ +{{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} +{{- if and .Values.createConfigmap $createConfigSecret }} +{{- $files := .Files }} +{{- $root := . -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ include "grafana.fullname" . }}-config-secret" + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- range $key, $value := .Values.alerting }} + {{- if (hasKey $value "secretFile") }} + {{- $key | nindent 2 }}: + {{- toYaml ( $files.Get $value.secretFile ) | b64enc | nindent 4}} + {{/* as of https://helm.sh/docs/chart_template_guide/accessing_files/ this will only work if you fork this chart and add files to it*/}} + {{- end }} +{{- end }} +stringData: +{{- range $key, $value := .Values.datasources }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} +{{ if (hasKey $value "secret") }} + {{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml new file mode 100644 index 0000000000..b412c4d1f0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.sidecar.dashboards.enabled .Values.sidecar.dashboards.SCProvider }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-config-dashboards + namespace: {{ include "grafana.namespace" . }} +data: + {{- include "grafana.configDashboardProviderData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap.yaml new file mode 100644 index 0000000000..7d7428be51 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap.yaml @@ -0,0 +1,15 @@ +{{- if .Values.createConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- include "grafana.configData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml new file mode 100644 index 0000000000..b96ce72026 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml @@ -0,0 +1,38 @@ +{{- if .Values.dashboards }} +{{ $files := .Files }} +{{- range $provider, $dashboards := .Values.dashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ include "grafana.namespace" $ }} + labels: + {{- include "grafana.labels" $ | nindent 4 }} + dashboard-provider: {{ $provider }} + {{- if $.Values.sidecar.dashboards.enabled }} + {{ $.Values.sidecar.dashboards.label }}: {{ $.Values.sidecar.dashboards.labelValue | quote }} + {{- end }} +{{- if $dashboards }} +data: +{{- $dashboardFound := false }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} +{{- $dashboardFound = true }} + {{- print $key | nindent 2 }}.json: + {{- if hasKey $value "json" }} + |- + {{- $value.json | nindent 6 }} + {{- end }} + {{- if hasKey $value "file" }} + {{- toYaml ( $files.Get $value.file ) | nindent 4}} + {{- end }} +{{- end }} +{{- end }} +{{- if not $dashboardFound }} + {} +{{- end }} +{{- end }} +--- +{{- end }} + +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/deployment.yaml new file mode 100644 index 0000000000..46c016faa3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/deployment.yaml @@ -0,0 +1,53 @@ +{{- if (and (not .Values.useStatefulSet) (or (not .Values.persistence.enabled) (eq .Values.persistence.type "pvc"))) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }} + replicas: {{ .Values.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + {{- with .Values.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include "grafana.configData" . | sha256sum }} + {{- if .Values.dashboards }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + {{- end }} + checksum/sc-dashboard-provider-config: {{ include "grafana.configDashboardProviderData" . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include "grafana.secretsData" . | sha256sum }} + {{- end }} + {{- if .Values.envRenderSecret }} + checksum/secret-env: {{ tpl (toYaml .Values.envRenderSecret) . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/extra-manifests.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/extra-manifests.yaml new file mode 100644 index 0000000000..a9bb3b6ba8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraObjects }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/headless-service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/headless-service.yaml new file mode 100644 index 0000000000..3028589d32 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/headless-service.yaml @@ -0,0 +1,22 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if or .Values.headlessService (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-headless + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + clusterIP: None + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + type: ClusterIP + ports: + - name: {{ .Values.gossipPortName }}-tcp + port: 9094 +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/hpa.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/hpa.yaml new file mode 100644 index 0000000000..46bbcb49a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/hpa.yaml @@ -0,0 +1,52 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if .Values.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }} + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + {{- if has .Values.persistence.type $sts }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ include "grafana.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.behavior }} + behavior: {{ toYaml .Values.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml new file mode 100644 index 0000000000..28231b803e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml @@ -0,0 +1,131 @@ +{{ if .Values.imageRenderer.enabled }} +{{- $root := . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.imageRenderer.autoscaling.enabled) (.Values.imageRenderer.replicas) }} + replicas: {{ .Values.imageRenderer.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + + {{- with .Values.imageRenderer.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 8 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.imageRenderer.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imageRenderer.schedulerName }} + schedulerName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.serviceAccountName }} + serviceAccountName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.image.pullSecrets }} + imagePullSecrets: + {{- range . }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + containers: + - name: {{ .Chart.Name }}-image-renderer + {{- $registry := include "system_default_registry" | default .Values.imageRenderer.image.registry -}} + {{- if .Values.imageRenderer.image.sha }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}@sha256:{{ .Values.imageRenderer.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.imageRenderer.image.pullPolicy }} + {{- if .Values.imageRenderer.command }} + command: + {{- range .Values.imageRenderer.command }} + - {{ . }} + {{- end }} + {{- end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + containerPort: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.imageRenderer.service.portName }} + env: + - name: HTTP_PORT + value: {{ .Values.imageRenderer.service.targetPort | quote }} + {{- if .Values.imageRenderer.serviceMonitor.enabled }} + - name: ENABLE_METRICS + value: "true" + {{- end }} + {{- range $key, $value := .Values.imageRenderer.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 16 }} + {{- end }} + {{- range $key, $value := .Values.imageRenderer.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- with .Values.imageRenderer.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /tmp + name: image-renderer-tmpfs + {{- with .Values.imageRenderer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.imageRenderer.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: image-renderer-tmpfs + emptyDir: {} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml new file mode 100644 index 0000000000..b0f0059b79 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "grafana.fullname" . }}-image-renderer + minReplicas: {{ .Values.imageRenderer.autoscaling.minReplicas }} + maxReplicas: {{ .Values.imageRenderer.autoscaling.maxReplicas }} + metrics: + {{- if .Values.imageRenderer.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.behavior }} + behavior: {{ toYaml .Values.imageRenderer.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml new file mode 100644 index 0000000000..d1a0eb313d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitIngress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-ingress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer ingress traffic from grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Ingress + ingress: + - ports: + - port: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.imageRenderer.networkPolicy.extraIngressSelectors -}} + {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} + +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitEgress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-egress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer egress traffic to grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Egress + egress: + # allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # talk only to grafana + - ports: + - port: {{ .Values.service.targetPort }} + protocol: TCP + to: + - namespaceSelector: + matchLabels: + name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-service.yaml new file mode 100644 index 0000000000..f8da127cf8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-service.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + {{- with .Values.imageRenderer.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + port: {{ .Values.imageRenderer.service.port }} + protocol: TCP + targetPort: {{ .Values.imageRenderer.service.targetPort }} + {{- with .Values.imageRenderer.appProtocol }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml new file mode 100644 index 0000000000..5d9f09d266 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if .Values.imageRenderer.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + {{- if .Values.imageRenderer.serviceMonitor.namespace }} + namespace: {{ tpl .Values.imageRenderer.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.imageRenderer.service.portName }} + {{- with .Values.imageRenderer.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.imageRenderer.serviceMonitor.path }} + scheme: {{ .Values.imageRenderer.serviceMonitor.scheme }} + {{- with .Values.imageRenderer.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}-image-renderer" + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.imageRenderer.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/ingress.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/ingress.yaml new file mode 100644 index 0000000000..b2ffd81095 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/ingress.yaml @@ -0,0 +1,78 @@ +{{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "grafana.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "grafana.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "grafana.ingress.supportsPathType" .) "true" -}} +{{- $fullName := include "grafana.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +{{- $extraPaths := .Values.ingress.extraPaths -}} +apiVersion: {{ include "grafana.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} + {{- with .Values.ingress.tls }} + tls: + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} + rules: + {{- if .Values.ingress.hosts }} + {{- range .Values.ingress.hosts }} + - host: {{ tpl . $ | quote }} + http: + paths: + {{- with $extraPaths }} + {{- toYaml . | nindent 10 }} + {{- end }} + - path: {{ $ingressPath }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end }} + {{- else }} + - http: + paths: + - backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- with $ingressPath }} + path: {{ . }} + {{- end }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/networkpolicy.yaml new file mode 100644 index 0000000000..4cd3ed6976 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/networkpolicy.yaml @@ -0,0 +1,61 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + policyTypes: + {{- if .Values.networkPolicy.ingress }} + - Ingress + {{- end }} + {{- if .Values.networkPolicy.egress.enabled }} + - Egress + {{- end }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + + {{- if .Values.networkPolicy.egress.enabled }} + egress: + {{- if not .Values.networkPolicy.egress.blockDNSResolution }} + - ports: + - port: 53 + protocol: UDP + {{- end }} + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- with .Values.networkPolicy.egress.to }} + to: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.networkPolicy.ingress }} + ingress: + - ports: + - port: {{ .Values.service.targetPort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ include "grafana.fullname" . }}-client: "true" + {{- with .Values.networkPolicy.explicitNamespacesSelector }} + - namespaceSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "grafana.labels" . | nindent 14 }} + role: read + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/nginx-config.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/nginx-config.yaml new file mode 100644 index 0000000000..557471f6ff --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/nginx-config.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-nginx-proxy-config + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 8080; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location /api/dashboards { + proxy_pass http://localhost:3000; + } + + location /api/search { + proxy_pass http://localhost:3000; + + sub_filter_types application/json; + sub_filter_once off; + } + + location /api/live/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $http_host; + proxy_pass http://localhost:3000; + } + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:3000/; + + sub_filter_once off; + + {{- if eq .Values.global.cattle.clusterId "local" -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- else -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- end -}} + + sub_filter ':"/avatar/' ':"avatar/'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000000..05251214ac --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: {{ include "grafana.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..973caccd57 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml @@ -0,0 +1,45 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + # Default set from Docker, with DAC_OVERRIDE and CHOWN + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'csi' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/pvc.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/pvc.yaml new file mode 100644 index 0000000000..c9b234305f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/pvc.yaml @@ -0,0 +1,41 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.persistence.extraPvcLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.finalizers }} + finalizers: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)) }} + volumeName: {{ (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)).spec.volumeName }} + {{- end }} + {{- with .Values.persistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/role.yaml new file mode 100644 index 0000000000..469b6f4e6c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/role.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)) }} +rules: + {{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}] + {{- end }} + {{- if and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled) }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- with .Values.rbac.extraRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/rolebinding.yaml new file mode 100644 index 0000000000..58f77c6b0b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/rolebinding.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + {{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} + {{- else }} + name: {{ include "grafana.fullname" . }} + {{- end }} +subjects: +- kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret-env.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret-env.yaml new file mode 100644 index 0000000000..eb14aac707 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret-env.yaml @@ -0,0 +1,14 @@ +{{- if .Values.envRenderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }}-env + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $key, $val := .Values.envRenderSecret }} + {{ $key }}: {{ tpl ($val | toString) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret.yaml new file mode 100644 index 0000000000..fd2ca50f4b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret.yaml @@ -0,0 +1,16 @@ +{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- include "grafana.secretsData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/service.yaml new file mode 100644 index 0000000000..e9396a15c6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/service.yaml @@ -0,0 +1,61 @@ +{{- if .Values.service.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} +spec: + {{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- with .Values.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- else if eq .Values.service.type "LoadBalancer" }} + type: LoadBalancer + {{- with .Values.service.loadBalancerIP }} + loadBalancerIP: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerClass }} + loadBalancerClass: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} + type: {{ .Values.service.type }} + {{- end }} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} + {{- with .Values.service.appProtocol }} + appProtocol: {{ . }} + {{- end }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- with .Values.extraExposePorts }} + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/serviceaccount.yaml new file mode 100644 index 0000000000..ffca0717ae --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.serviceAccount.autoMount | default .Values.serviceAccount.automountServiceAccountToken }} +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/servicemonitor.yaml new file mode 100644 index 0000000000..b321b1269c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/servicemonitor.yaml @@ -0,0 +1,68 @@ +{{- if .Values.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ tpl .Values.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- with .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + metricRelabelings: + {{- if .Values.serviceMonitor.metricRelabelings }} + {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.serviceMonitor.relabelings }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + {{- with .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}" + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/statefulset.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/statefulset.yaml new file mode 100644 index 0000000000..49278083e8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/statefulset.yaml @@ -0,0 +1,58 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if (or (.Values.useStatefulSet) (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)))}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + serviceName: {{ include "grafana.fullname" . }}-headless + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} + {{- if .Values.persistence.enabled}} + volumeClaimTemplates: + - metadata: + name: storage + spec: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + accessModes: {{ .Values.persistence.accessModes }} + storageClassName: {{ .Values.persistence.storageClassName }} + resources: + requests: + storage: {{ required "Must provide size for persistent volumes used by Grafana" .Values.persistence.size }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml new file mode 100644 index 0000000000..01c96c9243 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + run.sh: |- + @test "Test Health" { + url="http://{{ include "grafana.fullname" . }}/api/health" + + code=$(wget --server-response --spider --timeout 90 --tries 10 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml new file mode 100644 index 0000000000..70a0a884c9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -0,0 +1,32 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }}-test + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +spec: + allowPrivilegeEscalation: true + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + fsGroup: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - projected + - csi + - secret +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-role.yaml new file mode 100644 index 0000000000..976418b137 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-role.yaml @@ -0,0 +1,17 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}-test] +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml new file mode 100644 index 0000000000..509566eccd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "grafana.fullname" . }}-test +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml new file mode 100644 index 0000000000..38fba3596a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.testFramework.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test.yaml new file mode 100644 index 0000000000..83aaa185c2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test.yaml @@ -0,0 +1,53 @@ +{{- if .Values.testFramework.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + namespace: {{ include "grafana.namespace" . }} +spec: + serviceAccountName: {{ include "grafana.serviceAccountNameTest" . }} + {{- with .Values.testFramework.securityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 4 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 4 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: {{ .Release.Name }}-test + image: "{{ template "system_default_registry" . | default .Values.testFramework.image.registry }}/{{ .Values.testFramework.image.repository }}:{{ .Values.testFramework.image.tag }}" + imagePullPolicy: "{{ .Values.testFramework.imagePullPolicy}}" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + {{- with .Values.testFramework.resources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tests + configMap: + name: {{ include "grafana.fullname" . }}-test + restartPolicy: Never +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/values.yaml new file mode 100644 index 0000000000..45e3df9325 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/values.yaml @@ -0,0 +1,1315 @@ +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # Can be tempalted. + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + +rbac: + create: true + ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true) + # useExistingRole: name-of-some-role + # useExistingClusterRole: name-of-some-clusterRole + pspEnabled: false + pspUseAppArmor: false + namespaced: false + extraRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] + extraClusterRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] +serviceAccount: + create: true + name: + nameTest: + ## ServiceAccount labels. + labels: {} + ## Service account annotations. Can be templated. + # annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::123456789000:role/iam-role-name-here + + ## autoMount is deprecated in favor of automountServiceAccountToken + # autoMount: false + automountServiceAccountToken: true + +replicas: 1 + +## Create a headless service for the deployment +headlessService: false + +## Should the service account be auto mounted on the pod +automountServiceAccountToken: true + +## Create HorizontalPodAutoscaler object for deployment type +# +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + +## See `kubectl explain poddisruptionbudget.spec` for more +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} +# apiVersion: "" +# minAvailable: 1 +# maxUnavailable: 1 + +## See `kubectl explain deployment.spec.strategy` for more +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: RollingUpdate + +readinessProbe: + httpGet: + path: /api/health + port: 3000 + +livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: "default-scheduler" + +image: + repository: rancher/mirrored-grafana-grafana + # Overrides the Grafana image tag whose default is the chart appVersion + tag: 10.4.9 + sha: "" + pullPolicy: IfNotPresent + + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Can be templated. + ## + pullSecrets: [] + # - myRegistrKeySecretName + +testFramework: + enabled: false + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# dns configuration for pod +dnsPolicy: ~ +dnsConfig: {} + # nameservers: + # - 8.8.8.8 + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# Enable creating the grafana configmap +createConfigmap: true + +# Extra configmaps to mount in grafana pods +# Values are templated. +extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + + +extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + +# Apply extra labels to common labels. +extraLabels: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +downloadDashboardsImage: + repository: rancher/mirrored-curlimages-curl + tag: 7.85.0 + sha: "" + pullPolicy: IfNotPresent + +downloadDashboards: + env: {} + envFromSecret: "" + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## Pod Annotations +# podAnnotations: {} + +## Pod Labels +# podLabels: {} + +podPortName: grafana +gossipPortName: gossip +## Deployment annotations +# annotations: {} + +## Expose the grafana service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + enabled: true + type: ClusterIP + loadBalancerIP: "" + loadBalancerClass: "" + loadBalancerSourceRanges: [] + port: 80 + targetPort: 3000 + # targetPort: 4181 To be used with a proxy extraContainer + ## Service annotations. Can be templated. + annotations: {} + labels: {} + portName: service + # Adds the appProtocol field to the service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + +serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 30s + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + metricRelabelings: [] + targetLabels: [] + +extraExposePorts: [] + # - name: keycloak + # port: 8080 + # targetPort: 8080 + +# overrides pod.spec.hostAliases in the grafana deployment's pods +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "my.host.com" + +ingress: + enabled: false + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + # Values can be templated + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + path: / + + # pathType is only for k8s >= 1.1= + pathType: Prefix + + hosts: + - chart-example.local + ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## Or for k8s > 1.19 + # - path: /* + # pathType: Prefix + # backend: + # service: + # name: ssl-redirect + # port: + # name: use-annotation + + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Topology Spread Constraints +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## +topologySpreadConstraints: [] + +## Additional init containers (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +## +extraInitContainers: [] + +## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod +extraContainers: "" +# extraContainers: | +# - name: proxy +# image: quay.io/gambol99/keycloak-proxy:latest +# args: +# - -provider=github +# - -client-id= +# - -client-secret= +# - -github-org= +# - -email-domain=* +# - -cookie-secret= +# - -http-address=http://0.0.0.0:4181 +# - -upstream-url=http://127.0.0.1:3000 +# ports: +# - name: proxy-web +# containerPort: 4181 + +## Volumes that can be used in init containers that will not be mounted to deployment pods +extraContainerVolumes: [] +# - name: volume-from-secret +# secret: +# secretName: secret-to-mount +# - name: empty-dir-volume +# emptyDir: {} + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + type: pvc + enabled: false + # storageClassName: default + accessModes: + - ReadWriteOnce + size: 10Gi + # annotations: {} + finalizers: + - kubernetes.io/pvc-protection + # selectorLabels: {} + ## Sub-directory of the PV to mount. Can be templated. + # subPath: "" + ## Name of an existing PVC. Can be templated. + # existingClaim: + ## Extra labels to apply to a PVC. + extraPvcLabels: {} + + ## If persistence is not enabled, this allows to mount the + ## local storage in-memory to improve performance + ## + inMemory: + enabled: false + ## The maximum usage on memory medium EmptyDir would be + ## the minimum value between the SizeLimit specified + ## here and the sum of memory limits of all containers in a pod + ## + # sizeLimit: 300Mi + +initChownData: + ## If false, data ownership will not be reset at startup + ## This allows the grafana-server to be run with an arbitrary user + ## + enabled: true + + ## initChownData container image + ## + image: + repository: rancher/mirrored-library-busybox + tag: "1.31.1" + sha: "" + pullPolicy: IfNotPresent + + ## initChownData resource requests and limits + ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + securityContext: + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + capabilities: + add: + - CHOWN + +# Administrator credentials when not using an existing secret (see below) +adminUser: admin +# adminPassword: strongpassword + +# Use an existing secret for the admin user. +admin: + ## Name of the secret. Can be templated. + existingSecret: "" + userKey: admin-user + passwordKey: admin-password + +## Define command to be executed at startup by grafana container +## Needed if using `vault-env` to manage secrets (ref: https://banzaicloud.com/blog/inject-secrets-into-pods-vault/) +## Default is "run.sh" as defined in grafana's Dockerfile +# command: +# - "sh" +# - "/run.sh" + +## Optionally define args if command is used +## Needed if using `hashicorp/envconsul` to manage secrets +## By default no arguments are set +# args: +# - "-secret" +# - "secret/grafana" +# - "./grafana" + +## Extra environment variables that will be pass onto deployment pods +## +## to provide grafana with access to CloudWatch on AWS EKS: +## 1. create an iam role of type "Web identity" with provider oidc.eks.* (note the provider for later) +## 2. edit the "Trust relationships" of the role, add a line inside the StringEquals clause using the +## same oidc eks provider as noted before (same as the existing line) +## also, replace NAMESPACE and prometheus-operator-grafana with the service account namespace and name +## +## "oidc.eks.us-east-1.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:sub": "system:serviceaccount:NAMESPACE:prometheus-operator-grafana", +## +## 3. attach a policy to the role, you can use a built in policy called CloudWatchReadOnlyAccess +## 4. use the following env: (replace 123456789000 and iam-role-name-here with your aws account number and role name) +## +## env: +## AWS_ROLE_ARN: arn:aws:iam::123456789000:role/iam-role-name-here +## AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +## AWS_REGION: us-east-1 +## +## 5. uncomment the EKS section in extraSecretMounts: below +## 6. uncomment the annotation section in the serviceAccount: above +## make sure to replace arn:aws:iam::123456789000:role/iam-role-name-here with your role arn + +env: {} + +## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core +## Renders in container spec as: +## env: +## ... +## - name: +## valueFrom: +## +envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## The name of a secret in the same kubernetes namespace which contain values to be added to the environment +## This can be useful for auth tokens, etc. Value is templated. +envFromSecret: "" + +## Sensible environment variables that will be rendered as new secret object +## This can be useful for auth tokens, etc. +## If the secret values contains "{{", they'll need to be properly escaped so that they are not interpreted by Helm +## ref: https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function +envRenderSecret: {} + +## The names of secrets in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the secret must be defined with an optional key. +## Name is templated. +envFromSecrets: [] +## - name: secret-name +## prefix: prefix +## optional: true + +## The names of conifgmaps in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the configmap must be defined with an optional key. +## Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#configmapenvsource-v1-core +envFromConfigMaps: [] +## - name: configmap-name +## prefix: prefix +## optional: true + +# Inject Kubernetes services as environment variables. +# See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables +enableServiceLinks: true + +## Additional grafana server secret mounts +# Defines additional mounts with secrets. Secrets must be manually created in the namespace. +extraSecretMounts: [] + # - name: secret-files + # mountPath: /etc/secrets + # secretName: grafana-secret-files + # readOnly: true + # subPath: "" + # + # for AWS EKS (cloudwatch) use the following (see also instruction in env: above) + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # projected: + # defaultMode: 420 + # sources: + # - serviceAccountToken: + # audience: sts.amazonaws.com + # expirationSeconds: 86400 + # path: token + # + # for CSI e.g. Azure Key Vault use the following + # - name: secrets-store-inline + # mountPath: /run/secrets + # readOnly: true + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "akv-grafana-spc" + # nodePublishSecretRef: # Only required when using service principal mode + # name: grafana-akv-creds # Only required when using service principal mode + +## Additional grafana server volume mounts +# Defines additional volume mounts. +extraVolumeMounts: [] + # - name: extra-volume-0 + # mountPath: /mnt/volume0 + # readOnly: true + # - name: extra-volume-1 + # mountPath: /mnt/volume1 + # readOnly: true + # - name: grafana-secrets + # mountPath: /mnt/volume2 + +## Additional Grafana server volumes +extraVolumes: [] + # - name: extra-volume-0 + # existingClaim: volume-claim + # - name: extra-volume-1 + # hostPath: + # path: /usr/shared/ + # type: "" + # - name: grafana-secrets + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "grafana-env-spc" + +## Container Lifecycle Hooks. Execute a specific bash command or make an HTTP request +lifecycleHooks: {} + # postStart: + # exec: + # command: [] + +## Pass the plugins you want installed as a list. +## +plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + ## You can also use other plugin download URL, as long as they are valid zip files, + ## and specify the name of the plugin after the semicolon. Like this: + # - https://grafana.com/api/plugins/marcusolsson-json-datasource/versions/1.3.2/download;marcusolsson-json-datasource + +## Configure grafana datasources +## ref: http://docs.grafana.org/administration/provisioning/#datasources +## +datasources: {} +# datasources.yaml: +# apiVersion: 1 +# datasources: +# - name: Prometheus +# type: prometheus +# url: http://prometheus-prometheus-server +# access: proxy +# isDefault: true +# - name: CloudWatch +# type: cloudwatch +# access: proxy +# uid: cloudwatch +# editable: false +# jsonData: +# authType: default +# defaultRegion: us-east-1 +# deleteDatasources: [] +# - name: Prometheus + +## Configure grafana alerting (can be templated) +## ref: http://docs.grafana.org/administration/provisioning/#alerting +## +alerting: {} + # rules.yaml: + # apiVersion: 1 + # groups: + # - orgId: 1 + # name: '{{ .Chart.Name }}_my_rule_group' + # folder: my_first_folder + # interval: 60s + # rules: + # - uid: my_id_1 + # title: my_first_rule + # condition: A + # data: + # - refId: A + # datasourceUid: '-100' + # model: + # conditions: + # - evaluator: + # params: + # - 3 + # type: gt + # operator: + # type: and + # query: + # params: + # - A + # reducer: + # type: last + # type: query + # datasource: + # type: __expr__ + # uid: '-100' + # expression: 1==0 + # intervalMs: 1000 + # maxDataPoints: 43200 + # refId: A + # type: math + # dashboardUid: my_dashboard + # panelId: 123 + # noDataState: Alerting + # for: 60s + # annotations: + # some_key: some_value + # labels: + # team: sre_team_1 + # contactpoints.yaml: + # secret: + # apiVersion: 1 + # contactPoints: + # - orgId: 1 + # name: cp_1 + # receivers: + # - uid: first_uid + # type: pagerduty + # settings: + # integrationKey: XXX + # severity: critical + # class: ping failure + # component: Grafana + # group: app-stack + # summary: | + # {{ `{{ include "default.message" . }}` }} + +## Configure notifiers +## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels +## +notifiers: {} +# notifiers.yaml: +# notifiers: +# - name: email-notifier +# type: email +# uid: email1 +# # either: +# org_id: 1 +# # or +# org_name: Main Org. +# is_default: true +# settings: +# addresses: an_email_address@example.com +# delete_notifiers: + +## Configure grafana dashboard providers +## ref: http://docs.grafana.org/administration/provisioning/#dashboards +## +## `path` must be /var/lib/grafana/dashboards/ +## +dashboardProviders: {} +# dashboardproviders.yaml: +# apiVersion: 1 +# providers: +# - name: 'default' +# orgId: 1 +# folder: '' +# type: file +# disableDeletion: false +# editable: true +# options: +# path: /var/lib/grafana/dashboards/default + +## Configure grafana dashboard to import +## NOTE: To use dashboards you must also enable/configure dashboardProviders +## ref: https://grafana.com/dashboards +## +## dashboards per provider, use provider name as key. +## +dashboards: {} + # default: + # some-dashboard: + # json: | + # $RAW_JSON + # custom-dashboard: + # file: dashboards/custom-dashboard.json + # prometheus-stats: + # gnetId: 2 + # revision: 2 + # datasource: Prometheus + # local-dashboard: + # url: https://example.com/repository/test.json + # token: '' + # local-dashboard-base64: + # url: https://example.com/repository/test-b64.json + # token: '' + # b64content: true + # local-dashboard-gitlab: + # url: https://example.com/repository/test-gitlab.json + # gitlabToken: '' + # local-dashboard-bitbucket: + # url: https://example.com/repository/test-bitbucket.json + # bearerToken: '' + # local-dashboard-azure: + # url: https://example.com/repository/test-azure.json + # basic: '' + # acceptHeader: '*/*' + +## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. +## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. +## ConfigMap data example: +## +## data: +## example-dashboard.json: | +## RAW_JSON +## +dashboardsConfigMaps: {} +# default: "" + +## Grafana's primary configuration +## NOTE: values in map will be converted to ini format +## ref: http://docs.grafana.org/installation/configuration/ +## +grafana.ini: + paths: + data: /var/lib/grafana/ + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net + server: + domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ .Values.ingress.hosts | first }}{{ else }}''{{ end }}" +## grafana Authentication can be enabled with the following values on grafana.ini + # server: + # The full public facing url you use in browser, used for redirects and emails + # root_url: + # https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana + # auth.github: + # enabled: false + # allow_sign_up: false + # scopes: user:email,read:org + # auth_url: https://github.com/login/oauth/authorize + # token_url: https://github.com/login/oauth/access_token + # api_url: https://api.github.com/user + # team_ids: + # allowed_organizations: + # client_id: + # client_secret: +## LDAP Authentication can be enabled with the following values on grafana.ini +## NOTE: Grafana will fail to start if the value for ldap.toml is invalid + # auth.ldap: + # enabled: true + # allow_sign_up: true + # config_file: /etc/grafana/ldap.toml + +## Grafana's LDAP configuration +## Templated by the template in _helpers.tpl +## NOTE: To enable the grafana.ini must be configured with auth.ldap.enabled +## ref: http://docs.grafana.org/installation/configuration/#auth-ldap +## ref: http://docs.grafana.org/installation/ldap/#configuration +ldap: + enabled: false + # `existingSecret` is a reference to an existing secret containing the ldap configuration + # for Grafana in a key `ldap-toml`. + existingSecret: "" + # `config` is the content of `ldap.toml` that will be stored in the created secret + config: "" + # config: |- + # verbose_logging = true + + # [[servers]] + # host = "my-ldap-server" + # port = 636 + # use_ssl = true + # start_tls = false + # ssl_skip_verify = false + # bind_dn = "uid=%s,ou=users,dc=myorg,dc=com" + +## Grafana's SMTP configuration +## NOTE: To enable, grafana.ini must be configured with smtp.enabled +## ref: http://docs.grafana.org/installation/configuration/#smtp +smtp: + # `existingSecret` is a reference to an existing secret containing the smtp configuration + # for Grafana. + existingSecret: "" + userKey: "user" + passwordKey: "password" + +## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders +## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards +sidecar: + image: + repository: rancher/mirrored-kiwigrid-k8s-sidecar + tag: 1.26.1 + sha: "" + imagePullPolicy: IfNotPresent + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + # skipTlsVerify Set to true to skip tls verification for kube api calls + # skipTlsVerify: true + enableUniqueFilenames: false + readinessProbe: {} + livenessProbe: {} + # Log level default for all sidecars. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. Defaults to INFO + # logLevel: INFO + alerts: + enabled: false + # Additional environment variables for the alerts sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with alert are marked with + label: grafana_alert + # value of label that the configmaps with alert are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for alert config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/alerting/reload" + # Absolute path to shell script to execute after a alert got reloaded + script: null + skipReload: false + # This is needed if skipReload is true, to load any alerts defined at startup time. + # Deploy the alert sidecar as an initContainer. + initAlerts: false + # Additional alert sidecar volume mounts + extraMounts: [] + # Sets the size limit of the alert sidecar emptyDir volume + sizeLimit: {} + dashboards: + enabled: false + # Additional environment variables for the dashboards sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + SCProvider: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + # value of label that the configmaps with dashboards are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) + folder: /tmp/dashboards + # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead + defaultFolderName: null + # Namespaces list. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces. + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # If specified, the sidecar will look for annotation with this name to create folder and put graph here. + # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. + folderAnnotation: null + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/dashboards/reload" + # Absolute path to shell script to execute after a configmap got reloaded + script: null + skipReload: false + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # provider configuration that lets grafana manage the dashboards + provider: + # name of the provider, should be unique + name: sidecarProvider + # orgid as configured in grafana + orgid: 1 + # folder in which the dashboards should be imported in grafana + folder: '' + # type of the provider + type: file + # disableDelete to activate a import-only behaviour + disableDelete: false + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + # allow Grafana to replicate dashboard structure from filesystem + foldersFromFilesStructure: false + # Additional dashboard sidecar volume mounts + extraMounts: [] + # Sets the size limit of the dashboard sidecar emptyDir volume + sizeLimit: {} + datasources: + enabled: false + # Additional environment variables for the datasourcessidecar + env: {} + envValueFrom: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with datasources are marked with + label: grafana_datasource + # value of label that the configmaps with datasources are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for datasource config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload datasources + reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + # Absolute path to shell script to execute after a datasource got reloaded + script: null + skipReload: true + # This is needed if skipReload is true, to load any datasources defined at startup time. + # Deploy the datasources sidecar as an initContainer. + initDatasources: true + # Sets the size limit of the datasource sidecar emptyDir volume + sizeLimit: {} + plugins: + enabled: false + # Additional environment variables for the plugins sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with plugins are marked with + label: grafana_plugin + # value of label that the configmaps with plugins are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for plugin config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload plugins + reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload" + # Absolute path to shell script to execute after a plugin got reloaded + script: null + skipReload: false + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any plugins defined at startup time. + initPlugins: false + # Sets the size limit of the plugin sidecar emptyDir volume + sizeLimit: {} + notifiers: + enabled: false + # Additional environment variables for the notifierssidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with notifiers are marked with + label: grafana_notifier + # value of label that the configmaps with notifiers are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for notifier config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload notifiers + reloadURL: "http://localhost:3000/api/admin/provisioning/notifications/reload" + # Absolute path to shell script to execute after a notifier got reloaded + script: null + skipReload: false + # Deploy the notifier sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any notifiers defined at startup time. + initNotifiers: false + # Sets the size limit of the notifier sidecar emptyDir volume + sizeLimit: {} + +## Override the deployment namespace +## +namespaceOverride: "" + +## Number of old ReplicaSets to retain +## +revisionHistoryLimit: 10 + +## Add a seperate remote image renderer deployment/service +imageRenderer: + deploymentStrategy: {} + # Enable the image-renderer deployment & service + enabled: false + replicas: 1 + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + image: + # image-renderer Image repository + repository: rancher/mirrored-grafana-grafana-image-renderer + # image-renderer Image tag + tag: 3.10.5 + # image-renderer Image sha (optional) + sha: "" + # image-renderer ImagePullPolicy + pullPolicy: Always + # extra environment variables + env: + HTTP_HOST: "0.0.0.0" + # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758 + # RENDERING_MODE: clustered + # IGNORE_HTTPS_ERRORS: true + + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + + # image-renderer deployment serviceAccount + serviceAccountName: "" + # image-renderer deployment securityContext + securityContext: {} + # image-renderer deployment container securityContext + containerSecurityContext: + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ['ALL'] + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ## image-renderer pod annotation + podAnnotations: {} + # image-renderer deployment Host Aliases + hostAliases: [] + # image-renderer deployment priority class + priorityClassName: '' + service: + # Enable the image-renderer service + enabled: true + # image-renderer service port name + portName: 'http' + # image-renderer service port used by both service and deployment + port: 8081 + targetPort: 8081 + # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 1m + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + # See: https://doc.crds.dev/github.com/prometheus-operator/kube-prometheus/monitoring.coreos.com/ServiceMonitor/v1@v0.11.0#spec-targetLabels + targetLabels: [] + # - targetLabel1 + # - targetLabel2 + # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana + grafanaProtocol: http + # In case a sub_path is used this needs to be added to the image renderer callback + grafanaSubPath: "" + # name of the image-renderer port on the pod + podPortName: http + # number of image-renderer replica sets to keep + revisionHistoryLimit: 10 + networkPolicy: + # Enable a NetworkPolicy to limit inbound traffic to only the created grafana pods + limitIngress: true + # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods + limitEgress: false + # Allow additional services to access image-renderer (eg. Prometheus operator when ServiceMonitor is enabled) + extraIngressSelectors: [] + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment (evaluated as template) + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: "default-scheduler" + +networkPolicy: + ## @param networkPolicy.enabled Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to grafana port defined. + ## When true, grafana will accept connections from any source + ## (with the correct destination port). + ## + ingress: true + ## @param networkPolicy.ingress When true enables the creation + ## an ingress network policy + ## + allowExternal: true + ## @param networkPolicy.explicitNamespacesSelector A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed + ## If explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the grafana. + ## But sometimes, we want the grafana to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + ## + ## + ## + ## + ## + ## + egress: + ## @param networkPolicy.egress.enabled When enabled, an egress network policy will be + ## created allowing grafana to connect to external data sources from kubernetes cluster. + enabled: false + ## + ## @param networkPolicy.egress.blockDNSResolution When enabled, DNS resolution will be blocked + ## for all pods in the grafana namespace. + blockDNSResolution: false + ## + ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress + ports: [] + ## Add ports to the egress by specifying - port: + ## E.X. + ## - port: 80 + ## - port: 443 + ## + ## @param networkPolicy.egress.to Allow egress traffic to specific destinations + to: [] + ## Add destinations to the egress by specifying - ipBlock: + ## E.X. + ## to: + ## - namespaceSelector: + ## matchExpressions: + ## - {key: role, operator: In, values: [grafana]} + ## + ## + ## + ## + ## + +# Enable backward compatibility of kubernetes where version below 1.13 doesn't have the enableServiceLinks option +enableKubeBackwardCompatibility: false +useStatefulSet: false +# Create a dynamic manifests via values: +extraObjects: [] + # - apiVersion: "kubernetes-client.io/v1" + # kind: ExternalSecret + # metadata: + # name: grafana-secrets + # spec: + # backendType: gcpSecretsManager + # data: + # - key: grafana-admin-password + # name: adminPassword + +# assertNoLeakedSecrets is a helper function defined in _helpers.tpl that checks if secret +# values are not exposed in the rendered grafana.ini configmap. It is enabled by default. +# +# To pass values into grafana.ini without exposing them in a configmap, use variable expansion: +# https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion +# +# Alternatively, if you wish to allow secret values to be exposed in the rendered grafana.ini configmap, +# you can disable this check by setting assertNoLeakedSecrets to false. +assertNoLeakedSecrets: true diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/Chart.yaml new file mode 100644 index 0000000000..c81bc0093f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedKubelet +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/Chart.yaml new file mode 100644 index 0000000000..1efee00e2f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedNodeExporter +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/Chart.yaml new file mode 100644 index 0000000000..725e8f31b0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: k3sServer +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/Chart.yaml new file mode 100644 index 0000000000..002a6a180d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/Chart.yaml @@ -0,0 +1,32 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-kube-state-metrics +apiVersion: v2 +appVersion: 2.10.1 +description: Install kube-state-metrics to generate and expose cluster-level metrics +home: https://github.com/kubernetes/kube-state-metrics/ +keywords: +- metric +- monitoring +- prometheus +- kubernetes +maintainers: +- email: tariq.ibrahim@mulesoft.com + name: tariq1890 +- email: manuel@rueg.eu + name: mrueg +- email: david@0xdc.me + name: dotdc +name: kube-state-metrics +sources: +- https://github.com/kubernetes/kube-state-metrics/ +type: application +version: 5.16.4 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/README.md new file mode 100644 index 0000000000..843be89e69 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/README.md @@ -0,0 +1,85 @@ +# kube-state-metrics Helm Chart + +Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics). + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Migrating from stable/kube-state-metrics and kubernetes/kube-state-metrics + +You can upgrade in-place: + +1. [get repository info](#get-repository-info) +1. [upgrade](#upgrading-chart) your existing release name using the new chart repository + +## Upgrading to v3.0.0 + +v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side. + +The upgraded chart now the following changes: + +* Dropped support for helm v2 (helm v3 or later is required) +* collectors key was renamed to resources +* namespace key was renamed to namespaces + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-state-metrics +``` + +### kube-rbac-proxy + +You can enable `kube-state-metrics` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy one RBAC proxy container per endpoint (metrics & telemetry). +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics-read +rules: + - apiGroups: [ "" ] + resources: ["services/kube-state-metrics"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt new file mode 100644 index 0000000000..3589c24ec3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt @@ -0,0 +1,23 @@ +kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. +The exposed metrics can be found here: +https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics + +The metrics are exported on the HTTP endpoint /metrics on the listening port. +In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-state-metrics.namespace" . }}.svc.cluster.local:{{ .Values.service.port }}/metrics + +They are served either as plaintext or protobuf depending on the Accept header. +They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint. + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints are now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "kube-state-metrics.fullname" . }}"] + verbs: + - get +``` +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl new file mode 100644 index 0000000000..ed277fbb53 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl @@ -0,0 +1,196 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "kube-state-metrics.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kube-state-metrics.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kube-state-metrics.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "kube-state-metrics.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-state-metrics.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kube-state-metrics.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "kube-state-metrics.labels" }} +helm.sh/chart: {{ template "kube-state-metrics.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" . }} +{{- include "kube-state-metrics.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kube-state-metrics.selectorLabels" }} +{{- if .Values.selectorOverride }} +{{ toYaml .Values.selectorOverride }} +{{- else }} +app.kubernetes.io/name: {{ include "kube-state-metrics.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "kube-state-metrics.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +The image to use for kube-state-metrics +*/}} +{{- define "kube-state-metrics.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +The image to use for kubeRBACProxy +*/}} +{{- define "kubeRBACProxy.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.kubeRBACProxy.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..025cd47a88 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + endpointSelector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + egress: + {{- if and .Values.networkPolicy.cilium .Values.networkPolicy.cilium.kubeApiServerSelector }} + {{ toYaml .Values.networkPolicy.cilium.kubeApiServerSelector | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + - port: {{ .Values.service.port | quote }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + - port: {{ .Values.selfMonitor.telemetryPort | default 8081 | quote }} + protocol: TCP + {{ end }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..cf9f628d04 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rbac.useClusterRole -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml new file mode 100644 index 0000000000..d38a75a51d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.customResourceState.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config.yaml: | + {{- toYaml .Values.customResourceState.config | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml new file mode 100644 index 0000000000..03158eb948 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml @@ -0,0 +1,314 @@ +apiVersion: apps/v1 +{{- if .Values.autosharding.enabled }} +kind: StatefulSet +{{- else }} +kind: Deployment +{{- end }} +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: +{{ toYaml .Values.annotations | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + replicas: {{ .Values.replicas }} + {{- if not .Values.autosharding.enabled }} + strategy: + type: {{ .Values.updateStrategy | default "RollingUpdate" }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.autosharding.enabled }} + serviceName: {{ template "kube-state-metrics.fullname" . }} + volumeClaimTemplates: [] + {{- end }} + template: + metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 8 }} + {{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + spec: + hostNetwork: {{ .Values.hostNetwork }} + serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 6 }} + {{- end }} + containers: + {{- $servicePort := ternary 9090 (.Values.service.port | default 8080) .Values.kubeRBACProxy.enabled}} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - name: {{ template "kube-state-metrics.name" . }} + {{- if .Values.autosharding.enabled }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + args: + {{- if .Values.extraArgs }} + {{- .Values.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --port={{ $servicePort }} + {{- if .Values.collectors }} + - --resources={{ .Values.collectors | join "," }} + {{- end }} + {{- if .Values.metricLabelsAllowlist }} + - --metric-labels-allowlist={{ .Values.metricLabelsAllowlist | join "," }} + {{- end }} + {{- if .Values.metricAnnotationsAllowList }} + - --metric-annotations-allowlist={{ .Values.metricAnnotationsAllowList | join "," }} + {{- end }} + {{- if .Values.metricAllowlist }} + - --metric-allowlist={{ .Values.metricAllowlist | join "," }} + {{- end }} + {{- if .Values.metricDenylist }} + - --metric-denylist={{ .Values.metricDenylist | join "," }} + {{- end }} + {{- $namespaces := list }} + {{- if .Values.namespaces }} + {{- range $ns := join "," .Values.namespaces | split "," }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + {{- if .Values.releaseNamespace }} + {{- $namespaces = append $namespaces ( include "kube-state-metrics.namespace" . ) }} + {{- end }} + {{- if $namespaces }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + {{- if .Values.namespacesDenylist }} + - --namespaces-denylist={{ tpl (.Values.namespacesDenylist | join ",") $ }} + {{- end }} + {{- if .Values.autosharding.enabled }} + - --pod=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + {{- end }} + {{- if .Values.kubeconfig.enabled }} + - --kubeconfig=/opt/k8s/.kube/config + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - --telemetry-host=127.0.0.1 + - --telemetry-port={{ $telemetryPort }} + {{- else }} + {{- if .Values.selfMonitor.telemetryHost }} + - --telemetry-host={{ .Values.selfMonitor.telemetryHost }} + {{- end }} + {{- if .Values.selfMonitor.telemetryPort }} + - --telemetry-port={{ $telemetryPort }} + {{- end }} + {{- end }} + {{- if .Values.customResourceState.enabled }} + - --custom-resource-state-config-file=/etc/customresourcestate/config.yaml + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumeMounts) }} + volumeMounts: + {{- if .Values.kubeconfig.enabled }} + - name: kubeconfig + mountPath: /opt/k8s/.kube/ + readOnly: true + {{- end }} + {{- if .Values.customResourceState.enabled }} + - name: customresourcestate-config + mountPath: /etc/customresourcestate + readOnly: true + {{- end }} + {{- if .Values.volumeMounts }} +{{ toYaml .Values.volumeMounts | indent 8 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + image: {{ include "kube-state-metrics.image" . }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + {{- if .Values.selfMonitor.enabled }} + - containerPort: {{ $telemetryPort }} + name: "metrics" + {{- end }} + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: /healthz + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- if .Values.resources }} + resources: +{{ toYaml .Values.resources | indent 10 }} +{{- end }} +{{- if .Values.containerSecurityContext }} + securityContext: +{{ toYaml .Values.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-http + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port | default 8080}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.selfMonitor.enabled }} + - name: kube-rbac-proxy-telemetry + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.selfMonitor.telemetryPort | default 8081 }} + - --upstream=http://127.0.0.1:{{ $telemetryPort }}/ + - --proxy-endpoints-port=8889 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + name: "metrics" + - containerPort: 8889 + name: "metrics-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8889 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- end }} + {{- end }} + {{- with .Values.containers }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.topologySpreadConstraints | indent 8 }} + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumes) (.Values.kubeRBACProxy.enabled) }} + volumes: + {{- if .Values.kubeconfig.enabled}} + - name: kubeconfig + secret: + secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + {{- end }} + {{- if .Values.kubeRBACProxy.enabled}} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + {{- end }} + {{- if .Values.customResourceState.enabled}} + - name: customresourcestate-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + {{- end }} + {{- if .Values.volumes }} +{{ toYaml .Values.volumes | indent 8 }} + {{- end }} + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml new file mode 100644 index 0000000000..6af0084502 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.kubeconfig.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +type: Opaque +data: + config: '{{ .Values.kubeconfig.secret }}' +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml new file mode 100644 index 0000000000..309b38ec54 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "kubernetes") }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + {{- if .Values.networkPolicy.egress }} + ## Deny all egress by default + egress: + {{- toYaml .Values.networkPolicy.egress | nindent 4 }} + {{- end }} + ingress: + {{- if .Values.networkPolicy.ingress }} + {{- toYaml .Values.networkPolicy.ingress | nindent 4 }} + {{- else }} + ## Allow ingress on default ports by default + - ports: + - port: {{ .Values.service.port | default 8080 }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - port: {{ $telemetryPort }} + protocol: TCP + {{- end }} + {{- end }} + podSelector: + {{- if .Values.networkPolicy.podSelector }} + {{- toYaml .Values.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + policyTypes: + - Ingress + - Egress +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml new file mode 100644 index 0000000000..3771b511de --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podDisruptionBudget -}} +{{ if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" -}} +apiVersion: policy/v1 +{{- else -}} +apiVersion: policy/v1beta1 +{{- end }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..d9d944d740 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +{{- if .Values.podSecurityPolicy.annotations }} + annotations: +{{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }} +{{- end }} +spec: + privileged: false + volumes: + - 'secret' +{{- if .Values.podSecurityPolicy.additionalVolumes }} +{{ toYaml .Values.podSecurityPolicy.additionalVolumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..c69e01a716 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..df81c49028 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml new file mode 100644 index 0000000000..671dc9d660 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml @@ -0,0 +1,22 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "kube-state-metrics.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "kube-state-metrics.fullname" . }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/role.yaml new file mode 100644 index 0000000000..0170878376 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/role.yaml @@ -0,0 +1,215 @@ +{{- if not (kindIs "slice" .Values.collectors) }} +{{- fail "Collectors need to be a List since kube-state-metrics chart 3.2.2. Please check README for more information."}} +{{- end }} +{{- if and (eq .Values.rbac.create true) (not .Values.rbac.useExistingRole) -}} +{{- range (ternary (join "," .Values.namespaces | split "," ) (list "") (eq $.Values.rbac.useClusterRole false)) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +{{- if eq $.Values.rbac.useClusterRole false }} +kind: Role +{{- else }} +kind: ClusterRole +{{- end }} +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- if eq $.Values.rbac.useClusterRole false }} + namespace: {{ . }} +{{- end }} +rules: +{{ if has "certificatesigningrequests" $.Values.collectors }} +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] +{{ end -}} +{{ if has "configmaps" $.Values.collectors }} +- apiGroups: [""] + resources: + - configmaps + verbs: ["list", "watch"] +{{ end -}} +{{ if has "cronjobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - cronjobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "daemonsets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "deployments" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpoints" $.Values.collectors }} +- apiGroups: [""] + resources: + - endpoints + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpointslices" $.Values.collectors }} +- apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: ["list", "watch"] +{{ end -}} +{{ if has "horizontalpodautoscalers" $.Values.collectors }} +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "ingresses" $.Values.collectors }} +- apiGroups: ["extensions", "networking.k8s.io"] + resources: + - ingresses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "jobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "leases" $.Values.collectors }} +- apiGroups: ["coordination.k8s.io"] + resources: + - leases + verbs: ["list", "watch"] +{{ end -}} +{{ if has "limitranges" $.Values.collectors }} +- apiGroups: [""] + resources: + - limitranges + verbs: ["list", "watch"] +{{ end -}} +{{ if has "mutatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "namespaces" $.Values.collectors }} +- apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch"] +{{ end -}} +{{ if has "networkpolicies" $.Values.collectors }} +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: ["list", "watch"] +{{ end -}} +{{ if has "nodes" $.Values.collectors }} +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumeclaims" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumes" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "poddisruptionbudgets" $.Values.collectors }} +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "pods" $.Values.collectors }} +- apiGroups: [""] + resources: + - pods + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicasets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicationcontrollers" $.Values.collectors }} +- apiGroups: [""] + resources: + - replicationcontrollers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "resourcequotas" $.Values.collectors }} +- apiGroups: [""] + resources: + - resourcequotas + verbs: ["list", "watch"] +{{ end -}} +{{ if has "secrets" $.Values.collectors }} +- apiGroups: [""] + resources: + - secrets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "services" $.Values.collectors }} +- apiGroups: [""] + resources: + - services + verbs: ["list", "watch"] +{{ end -}} +{{ if has "statefulsets" $.Values.collectors }} +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "storageclasses" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "validatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "volumeattachments" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - volumeattachments + verbs: ["list", "watch"] +{{ end -}} +{{- if $.Values.kubeRBACProxy.enabled }} +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] +{{- end }} +{{- if $.Values.customResourceState.enabled }} +- apiGroups: ["apiextensions.k8s.io"] + resources: + - customresourcedefinitions + verbs: ["list", "watch"] +{{- end }} +{{ if $.Values.rbac.extraRules }} +{{ toYaml $.Values.rbac.extraRules }} +{{ end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml new file mode 100644 index 0000000000..330651b73f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.rbac.useClusterRole false) -}} +{{- range (join "," $.Values.namespaces) | split "," }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} + namespace: {{ . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role +{{- if (not $.Values.rbac.useExistingRole) }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- else }} + name: {{ $.Values.rbac.useExistingRole }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" $ }} + namespace: {{ template "kube-state-metrics.namespace" $ }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/service.yaml new file mode 100644 index 0000000000..6c486a662a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/service.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + annotations: + {{- if .Values.prometheusScrape }} + prometheus.io/scrape: '{{ .Values.prometheusScrape }}' + {{- end }} + {{- if .Values.service.annotations }} + {{- toYaml .Values.service.annotations | nindent 4 }} + {{- end }} +spec: + type: "{{ .Values.service.type }}" + ports: + - name: "http" + protocol: TCP + port: {{ .Values.service.port | default 8080}} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port | default 8080}} + {{ if .Values.selfMonitor.enabled }} + - name: "metrics" + protocol: TCP + port: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + targetPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- if .Values.selfMonitor.telemetryNodePort }} + nodePort: {{ .Values.selfMonitor.telemetryNodePort }} + {{- end }} + {{ end }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if .Values.autosharding.enabled }} + clusterIP: None +{{- else if .Values.service.clusterIP }} + clusterIP: "{{ .Values.service.clusterIP }}" +{{- end }} + selector: + {{- include "kube-state-metrics.selectorLabels" . | indent 4 }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml new file mode 100644 index 0000000000..38a93b31d1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- if .Values.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml new file mode 100644 index 0000000000..01ec44e067 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -0,0 +1,126 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + {{- with .Values.prometheus.monitor.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- with .Values.prometheus.monitor.targetLabels }} + targetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | indent 2 }} + {{- if .Values.prometheus.monitor.namespaceSelector }} + namespaceSelector: + matchNames: + {{- with .Values.prometheus.monitor.namespaceSelector }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + endpoints: + - port: http + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + metricRelabelings: + {{- if .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml .Values.prometheus.monitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.selfMonitor.enabled }} + - port: metrics + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml new file mode 100644 index 0000000000..489de147c1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} + resources: + - statefulsets + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml new file mode 100644 index 0000000000..73b37a4f64 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..f46305b517 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml @@ -0,0 +1,44 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-state-metrics.name" . }} + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{ toYaml .Values.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{ toYaml .Values.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + {{- if .Values.autosharding.enabled }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ template "kube-state-metrics.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/values.yaml new file mode 100644 index 0000000000..bc8ee28fda --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/values.yaml @@ -0,0 +1,491 @@ +# Default values for kube-state-metrics. +prometheusScrape: true +image: + registry: docker.io + repository: rancher/mirrored-kube-state-metrics-kube-state-metrics + tag: v2.10.1 + sha: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: "image-pull-secret" + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "" + +# If set to true, this will deploy kube-state-metrics as a StatefulSet and the data +# will be automatically sharded across <.Values.replicas> pods using the built-in +# autodiscovery feature: https://github.com/kubernetes/kube-state-metrics#automated-sharding +# This is an experimental feature and there are no stability guarantees. +autosharding: + enabled: false + +replicas: 1 + +# Change the deployment strategy when autosharding is disabled. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +# The default is "RollingUpdate" as per Kubernetes defaults. +# During a release, 'RollingUpdate' can lead to two running instances for a short period of time while 'Recreate' can create a small gap in data. +# updateStrategy: Recreate + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +# List of additional cli arguments to configure kube-state-metrics +# for example: --enable-gzip-encoding, --log-file, etc. +# all the possible args can be found here: https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md +extraArgs: [] + +service: + port: 8080 + # Default to clusterIP for backward compatibility + type: ClusterIP + nodePort: 0 + loadBalancerIP: "" + # Only allow access to the loadBalancerIP from these IPs + loadBalancerSourceRanges: [] + clusterIP: "" + annotations: {} + +## Additional labels to add to all resources +customLabels: {} + # app: kube-state-metrics + +## Override selector labels +selectorOverride: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +hostNetwork: false + +rbac: + # If true, create & use RBAC resources + create: true + + # Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to it, rolename set here. + # useExistingRole: your-existing-role + + # If set to false - Run without Cluteradmin privs needed - ONLY works if namespace is also set (if useExistingRole is set this name is used as ClusterRole or Role to bind to) + useClusterRole: true + + # Add permissions for CustomResources' apiGroups in Role/ClusterRole. Should be used in conjunction with Custom Resource State Metrics configuration + # Example: + # - apiGroups: ["monitoring.coreos.com"] + # resources: ["prometheuses"] + # verbs: ["list", "watch"] + extraRules: [] + +# Configure kube-rbac-proxy. When enabled, creates one kube-rbac-proxy container per exposed HTTP endpoint (metrics and telemetry if enabled). +# The requests are served through the same service but requests are then HTTPS. +kubeRBACProxy: + enabled: false + image: + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.14.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-prxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + + ## volumeMounts enables mounting custom volumes in rbac-proxy containers + ## Useful for TLS certificates and keys + volumeMounts: [] + # - mountPath: /etc/tls + # name: kube-rbac-proxy-tls + # readOnly: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created, require rbac true + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # Reference to one or more secrets to be used when pulling images + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + imagePullSecrets: [] + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +prometheus: + monitor: + enabled: false + annotations: {} + additionalLabels: {} + namespace: "" + namespaceSelector: [] + jobLabel: "" + targetLabels: [] + podTargetLabels: [] + interval: "" + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + scrapeTimeout: "" + proxyUrl: "" + ## Whether to enable HTTP2 for servicemonitor + # enableHttp2: false + selectorOverride: {} + honorLabels: false + metricRelabelings: [] + relabelings: [] + scheme: "" + ## File to read bearer token for scraping targets + bearerTokenFile: "" + ## Secret to mount to read bearer token for scraping targets. The secret needs + ## to be in the same namespace as the service monitor and accessible by the + ## Prometheus Operator + bearerTokenSecret: {} + # name: secret-name + # key: key-name + tlsConfig: {} + +## Specify if a Pod Security Policy for kube-state-metrics must be created +## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +podSecurityPolicy: + annotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + additionalVolumes: [] + +## Configure network policy for kube-state-metrics +networkPolicy: + enabled: false + # networkPolicy.flavor -- Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + ## Configure the cilium network policy kube-apiserver selector + # cilium: + # kubeApiServerSelector: + # - toEntities: + # - kube-apiserver + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app.kubernetes.io/name: kube-state-metrics + +securityContext: + enabled: true + runAsGroup: 65534 + runAsUser: 65534 + fsGroup: 65534 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +## Specify security settings for a Container +## Allows overrides and additional options compared to (Pod) securityContext +## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +## Affinity settings for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ +affinity: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +## Topology spread constraints for pod assignment +## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +topologySpreadConstraints: [] + +# Annotations to be added to the deployment/statefulset +annotations: {} + +# Annotations to be added to the pod +podAnnotations: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +# Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} + +# Comma-separated list of metrics to be exposed. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricAllowlist: [] + +# Comma-separated list of metrics not to be enabled. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricDenylist: [] + +# Comma-separated list of additional Kubernetes label keys that will be used in the resource's +# labels metric. By default the metric contains only name and namespace labels. +# To include additional labels, provide a list of resource names in their plural form and Kubernetes +# label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. +# A single '*' can be provided per resource instead to allow any labels, but that has +# severe performance implications (Example: '=pods=[*]'). +metricLabelsAllowlist: [] + # - namespaces=[k8s-label-1,k8s-label-n] + +# Comma-separated list of Kubernetes annotations keys that will be used in the resource' +# labels metric. By default the metric contains only name and namespace labels. +# To include additional annotations provide a list of resource names in their plural form and Kubernetes +# annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. +# A single '*' can be provided per resource instead to allow any annotations, but that has +# severe performance implications (Example: '=pods=[*]'). +metricAnnotationsAllowList: [] + # - pods=[k8s-annotation-1,k8s-annotation-n] + +# Available collectors for kube-state-metrics. +# By default, all available resources are enabled, comment out to disable. +collectors: + - certificatesigningrequests + - configmaps + - cronjobs + - daemonsets + - deployments + - endpoints + - horizontalpodautoscalers + - ingresses + - jobs + - leases + - limitranges + - mutatingwebhookconfigurations + - namespaces + - networkpolicies + - nodes + - persistentvolumeclaims + - persistentvolumes + - poddisruptionbudgets + - pods + - replicasets + - replicationcontrollers + - resourcequotas + - secrets + - services + - statefulsets + - storageclasses + - validatingwebhookconfigurations + - volumeattachments + +# Enabling kubeconfig will pass the --kubeconfig argument to the container +kubeconfig: + enabled: false + # base64 encoded kube-config file + secret: + +# Enabling support for customResourceState, will create a configMap including your config that will be read from kube-state-metrics +customResourceState: + enabled: false + # Add (Cluster)Role permissions to list/watch the customResources defined in the config to rbac.extraRules + config: {} + +# Enable only the release namespace for collecting resources. By default all namespaces are collected. +# If releaseNamespace and namespaces are both set a merged list will be collected. +releaseNamespace: false + +# Comma-separated list(string) or yaml list of namespaces to be enabled for collecting resources. By default all namespaces are collected. +namespaces: "" + +# Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, +# only namespaces that are excluded in namespaces-denylist will be used. +namespacesDenylist: "" + +## Override the deployment namespace +## +namespaceOverride: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +## Provide a k8s version to define apiGroups for podSecurityPolicy Cluster Role. +## For example: kubeTargetVersionOverride: 1.14.9 +## +kubeTargetVersionOverride: "" + +# Enable self metrics configuration for service and Service Monitor +# Default values for telemetry configuration can be overridden +# If you set telemetryNodePort, you must also set service.type to NodePort +selfMonitor: + enabled: false + # telemetryHost: 0.0.0.0 + # telemetryPort: 8081 + # telemetryNodePort: 0 + +# Enable vertical pod autoscaler support for kube-state-metrics +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: [] + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# volumeMounts are used to add custom volume mounts to deployment. +# See example below +volumeMounts: [] +# - mountPath: /etc/config +# name: config-volume + +# volumes are used to add custom volumes to deployment +# See example below +volumes: [] +# - configMap: +# name: cm-for-volume +# name: config-volume + +# Extra manifests to deploy as an array +extraManifests: [] + # - apiVersion: v1 + # kind: ConfigMap + # metadata: + # labels: + # name: prometheus-extra + # data: + # extra-data: "value" + +## Containers allows injecting additional containers. +containers: [] + # - name: crd-init + # image: kiwigrid/k8s-sidecar:latest + +## InitContainers allows injecting additional initContainers. +initContainers: [] + # - name: crd-sidecar + # image: kiwigrid/k8s-sidecar:latest + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml new file mode 100644 index 0000000000..0b753af5db --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmControllerManager +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/Chart.yaml new file mode 100644 index 0000000000..389e1386b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmEtcd +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/Chart.yaml new file mode 100644 index 0000000000..b755e8cd9e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmProxy +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/Chart.yaml new file mode 100644 index 0000000000..f7c201833a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmScheduler +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/Chart.yaml new file mode 100644 index 0000000000..d067725a17 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/Chart.yaml @@ -0,0 +1,28 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-prometheus-adapter +apiVersion: v1 +appVersion: v0.10.0 +description: A Helm chart for k8s prometheus adapter +home: https://github.com/kubernetes-sigs/prometheus-adapter +keywords: +- hpa +- metrics +- prometheus +- adapter +kubeVersion: '>=1.26.0-0' +maintainers: +- email: mattias.gees@jetstack.io + name: mattiasgees +- name: steven-sheehy +- email: hfernandez@mesosphere.com + name: hectorj2f +name: prometheus-adapter +sources: +- https://github.com/kubernetes/charts +- https://github.com/kubernetes-sigs/prometheus-adapter +version: 4.2.0 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/README.md new file mode 100644 index 0000000000..d77bb0c920 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/README.md @@ -0,0 +1,160 @@ +# Prometheus Adapter + +Installs the [Prometheus Adapter](https://github.com/kubernetes-sigs/prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics. + +## Prerequisites + +Kubernetes 1.14+ + +## Get Helm Repositories Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-adapter +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Helm Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### To 4.2.0 + +Readiness and liveness probes are now fully configurable through values `readinessProbe` and `livenessProbe`. The previous values have been kept as defaults. + +### To 4.0.0 + +Previously, security context of the container was set directly in the deployment template. This release makes it configurable through the new configuration variable `securityContext` whilst keeping the previously set values as defaults. Furthermore, previous variable `runAsUser` is now set in `securityContext` and is not used any longer. Please, use `securityContext.runAsUser` instead. In the same security context, `seccompProfile` has been enabled and set to type `RuntimeDefault`. + +### To 3.0.0 + +Due to a change in deployment labels, the upgrade requires `helm upgrade --force` in order to re-create the deployment. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-adapter +``` + +### Prometheus Service Endpoint + +To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`. + +### Adapter Rules + +Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/kubernetes-sigs/prometheus-adapter/blob/master/docs/config.md) for the proper format). + +### Horizontal Pod Autoscaler Metrics + +Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics). + +The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis): + +### Custom Metrics + +Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`: + +```yaml +rules: + custom: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### External Metrics + +Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`: + +```yaml +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_external_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### Resource Metrics + +Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`: + +```yaml +rules: + resource: + cpu: + containerQuery: | + sum by (<<.GroupBy>>) ( + rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + - + avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m +``` + +**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt new file mode 100644 index 0000000000..b7b9b99322 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt @@ -0,0 +1,9 @@ +{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed. +In a few minutes you should be able to list metrics using the following command(s): +{{ if .Values.rules.resource }} + kubectl get --raw /apis/metrics.k8s.io/v1beta1 +{{- end }} + kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 +{{ if .Values.rules.external }} + kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl new file mode 100644 index 0000000000..edbb829b2b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl @@ -0,0 +1,113 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-prometheus-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-prometheus-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "k8s-prometheus-adapter.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-prometheus-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "k8s-prometheus-adapter.labels" }} +helm.sh/chart: {{ include "k8s-prometheus-adapter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "k8s-prometheus-adapter.name" . }} +{{- include "k8s-prometheus-adapter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "k8s-prometheus-adapter.selectorLabels" }} +app.kubernetes.io/name: {{ include "k8s-prometheus-adapter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8s-prometheus-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "k8s-prometheus-adapter.pdb.apiVersion" -}} +{{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} + {{- print "policy/v1" -}} +{{- else -}} + {{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml new file mode 100644 index 0000000000..4e32c964c6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml @@ -0,0 +1,76 @@ +{{- if .Values.certManager.enabled -}} +--- +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + duration: {{ .Values.certManager.caCertDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.prometheus-adapter" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + ca: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert +--- +# Finally, generate a serving certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + duration: {{ .Values.certManager.certDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + dnsNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }}.svc +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml new file mode 100644 index 0000000000..6701e6ba08 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-system-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml new file mode 100644 index 0000000000..67efd2aa2f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml new file mode 100644 index 0000000000..2c690a03cc --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml new file mode 100644 index 0000000000..17f415d970 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml @@ -0,0 +1,97 @@ +{{- if not .Values.rules.existing -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +data: + config.yaml: | +{{- if or .Values.rules.default .Values.rules.custom }} + rules: +{{- if .Values.rules.default }} + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{- end -}} +{{- if .Values.rules.custom }} +{{ toYaml .Values.rules.custom | indent 4 }} +{{- end -}} +{{- end -}} +{{- if .Values.rules.external }} + externalRules: +{{ toYaml .Values.rules.external | indent 4 }} +{{- end -}} +{{- if .Values.rules.resource }} + resourceRules: +{{ toYaml .Values.rules.resource | indent 6 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml new file mode 100644 index 0000000000..8b7b4e555e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if or .Values.rules.default .Values.rules.custom }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: custom.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..0cc6920836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,24 @@ +{{- /* +This if must be aligned with custom-metrics-cluster-role.yaml +as otherwise this binding will point to not existing role. +*/ -}} +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml new file mode 100644 index 0000000000..4aa15ffe99 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: ["*"] + verbs: ["*"] +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml new file mode 100644 index 0000000000..a7ea3310a0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml @@ -0,0 +1,143 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + {{- if or .Values.customAnnotations .Values.deploymentAnnotations }} + annotations: + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.deploymentAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + replicas: {{ .Values.replicas }} + strategy: {{ toYaml .Values.strategy | nindent 4 }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} + template: + metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + name: {{ template "k8s-prometheus-adapter.name" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy }} + {{- end}} + containers: + - name: {{ .Chart.Name }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 8 }} + {{- end }} + args: + - /adapter + - --secure-port={{ .Values.listenPort }} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - --tls-cert-file=/var/run/serving-cert/tls.crt + - --tls-private-key-file=/var/run/serving-cert/tls.key + {{- end }} + - --cert-dir=/tmp/cert + - --prometheus-url={{ tpl .Values.prometheus.url . }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}}{{ .Values.prometheus.path }} + - --metrics-relist-interval={{ .Values.metricsRelistInterval }} + - --v={{ .Values.logLevel }} + - --config=/etc/adapter/config.yaml + {{- if .Values.extraArguments }} + {{- toYaml .Values.extraArguments | trim | nindent 8 }} + {{- end }} + ports: + - containerPort: {{ .Values.listenPort }} + name: https + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }} + {{ end }} + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} + priorityClassName: {{ .Values.priorityClassName }} + {{- if .Values.podSecurityContext }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | trim | nindent 6 }} + {{ end }} + - name: config + configMap: + name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }} + - name: tmp + emptyDir: {} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - name: volume-serving-cert + secret: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml new file mode 100644 index 0000000000..21339af128 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.external }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: external.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..05547bd323 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml new file mode 100644 index 0000000000..212ea78b25 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml new file mode 100644 index 0000000000..205761a9f1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml @@ -0,0 +1,23 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: {{ include "k8s-prometheus-adapter.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/psp.yaml new file mode 100644 index 0000000000..fded5a7491 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/psp.yaml @@ -0,0 +1,66 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.psp.create) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + hostPorts: + - min: {{ .Values.listenPort }} + max: {{ .Values.listenPort }} + {{- end }} + fsGroup: + rule: RunAsAny + runAsGroup: + rule: RunAsAny + runAsUser: + rule: MustRunAs + ranges: + - min: 1024 + max: 65535 + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - secret + - emptyDir + - configMap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +rules: +- apiGroups: + - 'policy' + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml new file mode 100644 index 0000000000..0cc9fff6a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.resource}} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml new file mode 100644 index 0000000000..3c247e48d2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml new file mode 100644 index 0000000000..73d8953046 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml new file mode 100644 index 0000000000..d3c77c1c65 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/secret.yaml new file mode 100644 index 0000000000..3e7e8887bd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- if .Values.tls.enable -}} +apiVersion: v1 +kind: Secret +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +type: kubernetes.io/tls +data: + tls.crt: {{ b64enc .Values.tls.certificate }} + tls.key: {{ b64enc .Values.tls.key }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/service.yaml new file mode 100644 index 0000000000..ddac37cfa1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + {{- if or .Values.service.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.service.annotations }} + {{ toYaml .Values.service.annotations | indent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + ports: + - port: {{ .Values.service.port }} + protocol: TCP + targetPort: https + selector: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 4 }} + type: {{ .Values.service.type }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..30a169ae0e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +{{- if or .Values.serviceAccount.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.serviceAccount.annotations }} + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/values.yaml new file mode 100644 index 0000000000..a1445a23f1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/values.yaml @@ -0,0 +1,277 @@ +# Default values for k8s-prometheus-adapter.. +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +affinity: {} + +topologySpreadConstraints: [] + +image: + repository: rancher/mirrored-prometheus-adapter-prometheus-adapter + tag: v0.12.0 + pullPolicy: IfNotPresent + +logLevel: 4 + +metricsRelistInterval: 1m + +listenPort: 6443 + +nodeSelector: {} + +priorityClassName: "" + +## Override the release namespace (for multi-namespace deployments in combined charts) +namespaceOverride: "" + +## Additional annotations to add to all resources +customAnnotations: {} + # role: custom-metrics + +## Additional labels to add to all resources +customLabels: {} + # monitoring: prometheus-adapter + +# Url to access prometheus +prometheus: + # Value is templated + url: http://prometheus.default.svc + port: 9090 + path: "" + +replicas: 1 + +# k8s 1.21 needs fsGroup to be set for non root deployments +# ref: https://github.com/kubernetes/kubernetes/issues/70679 +podSecurityContext: + fsGroup: 10001 + +# SecurityContext of the container +# ref. https://kubernetes.io/docs/tasks/configure-pod-container/security-context +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["all"] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + +rbac: + # Specifies whether RBAC resources should be created + create: true + +psp: + # Specifies whether PSP resources should be created + create: false + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +# Custom DNS configuration to be added to prometheus-adapter pods +dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + +# Configure liveness probe +# https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#Probe +livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +# Configure readiness probe +readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +rules: + default: true + + custom: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_custom_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # Mounts a configMap with pre-generated rules for use. Overrides the + # default, custom, external and resource entries + existing: + + external: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_external_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # resource: + # cpu: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # memory: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + # - + # avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # window: 3m + +service: + annotations: {} + port: 443 + type: ClusterIP + # clusterIP: 1.2.3.4 + +tls: + enable: false + ca: |- + # Public CA file that signed the APIService + key: |- + # Private key of the APIService + certificate: |- + # Public key of the APIService + +# Set environment variables from secrets, configmaps or by setting them as name/value +env: [] + # - name: TMP_DIR + # value: /tmp + # - name: PASSWORD + # valueFrom: + # secretKeyRef: + # name: mysecret + # key: password + # optional: false + +# Any extra arguments +extraArguments: [] + # - --tls-private-key-file=/etc/tls/tls.key + # - --tls-cert-file=/etc/tls/tls.crt + +# Any extra volumes +extraVolumes: [] + # - name: example-name + # hostPath: + # path: /path/on/host + # type: DirectoryOrCreate + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + # type: File + +# Any extra volume mounts +extraVolumeMounts: [] + # - name: example-name + # mountPath: /path/in/container + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +tolerations: [] + +# Labels added to the pod +podLabels: {} + +# Annotations added to the pod +podAnnotations: {} + +# Annotations added to the deployment +deploymentAnnotations: {} + +hostNetwork: + # Specifies if prometheus-adapter should be started in hostNetwork mode. + # + # You would require this enabled if you use alternate overlay networking for pods and + # API server unable to communicate with metrics-server. As an example, this is required + # if you use Weave network on EKS. See also dnsPolicy + enabled: false + +# When hostNetwork is enabled, you probably want to set this to ClusterFirstWithHostNet +# dnsPolicy: ClusterFirstWithHostNet + +# Deployment strategy type +strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% + +podDisruptionBudget: + # Specifies if PodDisruptionBudget should be enabled + # When enabled, minAvailable or maxUnavailable should also be defined. + enabled: false + minAvailable: + maxUnavailable: 1 + +certManager: + enabled: false + caCertDuration: 43800h + certDuration: 8760h diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/Chart.yaml new file mode 100644 index 0000000000..9130cbcc91 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/Chart.yaml @@ -0,0 +1,25 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: |- + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts +apiVersion: v2 +appVersion: 1.7.0 +description: A Helm chart for prometheus node-exporter +home: https://github.com/prometheus/node_exporter/ +keywords: +- node-exporter +- prometheus +- exporter +maintainers: +- email: gianrubio@gmail.com + name: gianrubio +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rootsandtrees@posteo.de + name: zeritti +name: prometheus-node-exporter +sources: +- https://github.com/prometheus/node_exporter/ +type: application +version: 4.30.3 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/README.md new file mode 100644 index 0000000000..149b982267 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/README.md @@ -0,0 +1,97 @@ + +# Prometheus Node Exporter + +Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a Prometheus [Node Exporter](http://github.com/prometheus/node_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-node-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/prometheus-node-exporter --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### 3.x to 4.x + +Starting from version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i prometheus-node-exporter prometheus-community/prometheus-node-exporter +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 2.x to 3.x + +Change the following: + +```yaml +hostRootFsMount: true +``` + +to: + +```yaml +hostRootFsMount: + enabled: true + mountPropagation: HostToContainer +``` + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-node-exporter +``` + +### kube-rbac-proxy + +You can enable `prometheus-node-exporter` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy a RBAC proxy container protecting the node-exporter endpoint. +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-node-exporter-read +rules: + - apiGroups: [ "" ] + resources: ["services/node-exporter-prometheus-node-exporter"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt new file mode 100644 index 0000000000..8c5391f1f7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt @@ -0,0 +1,29 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus-node-exporter.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "prometheus-node-exporter.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "prometheus-node-exporter.namespace" . }} {{ template "prometheus-node-exporter.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "prometheus-node-exporter.namespace" . }} -l "app.kubernetes.io/name={{ template "prometheus-node-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:{{ .Values.service.port }} to use your application" + kubectl port-forward --namespace {{ template "prometheus-node-exporter.namespace" . }} $POD_NAME {{ .Values.service.port }} +{{- end }} + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints is now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "prometheus-node-exporter.fullname" . }}"] + verbs: + - get +``` +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl new file mode 100644 index 0000000000..72a6db45a1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -0,0 +1,236 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus-node-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus-node-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-node-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-node-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-node-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-node-exporter.name" . }} +{{ include "prometheus-node-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-node-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-node-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-node-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-node-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-node-exporter.image" -}} +{{- $temp_registry := (include "system_default_registry" .) }} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if $temp_registry }} +{{- printf "%s%s:%s@%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if $temp_registry }} +{{- printf "%s%s:%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-node-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-node-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-node-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-node-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* Sets sidecar volumeMounts */}} +{{- define "prometheus-node-exporter.sidecarVolumeMounts" -}} +{{- range $_, $mount := $.Values.sidecarVolumeMount }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- end }} +{{- range $_, $mount := $.Values.sidecarHostVolumeMounts }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- if $mount.mountPropagation }} + mountPropagation: {{ $mount.mountPropagation }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml new file mode 100644 index 0000000000..c256dba73d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: + {{- if $.Values.kubeRBACProxy.enabled }} + - apiGroups: [ "authentication.k8s.io" ] + resources: + - tokenreviews + verbs: [ "create" ] + - apiGroups: [ "authorization.k8s.io" ] + resources: + - subjectaccessreviews + verbs: [ "create" ] + {{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..653305ad9e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + name: {{ template "prometheus-node-exporter.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml new file mode 100644 index 0000000000..48d274f1b1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -0,0 +1,309 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ ternary true false (or .Values.serviceAccount.automountServiceAccountToken .Values.kubeRBACProxy.enabled) }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.extraInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-node-exporter.serviceAccountName" . }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + containers: + {{- $servicePort := ternary .Values.kubeRBACProxy.port .Values.service.port .Values.kubeRBACProxy.enabled }} + - name: node-exporter + image: {{ include "prometheus-node-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + {{- if .Values.hostRootFsMount.enabled }} + - --path.rootfs=/host/root + {{- if semverCompare ">=1.4.0-0" (coalesce .Values.version .Values.image.tag .Chart.AppVersion) }} + - --path.udev.data=/host/root/run/udev/data + {{- end }} + {{- end }} + - --web.listen-address=[$(HOST_IP)]:{{ $servicePort }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: HOST_IP + {{- if .Values.kubeRBACProxy.enabled }} + value: 127.0.0.1 + {{- else if .Values.service.listenOnAllInterfaces }} + value: 0.0.0.0 + {{- else }} + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + {{- end }} + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - name: {{ .Values.service.portName }} + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + volumeMounts: + - name: proc + mountPath: /host/proc + {{- with .Values.hostProcFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + - name: sys + mountPath: /host/sys + {{- with .Values.hostSysFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- if .Values.hostRootFsMount.enabled }} + - name: root + mountPath: /host/root + {{- with .Values.hostRootFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- with $mount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- range .Values.sidecars }} + {{- $overwrites := dict "volumeMounts" (concat (include "prometheus-node-exporter.sidecarVolumeMounts" $ | fromYamlArray) (.volumeMounts | default list) | default list) }} + {{- $defaults := dict "image" (include "prometheus-node-exporter.image" $) "securityContext" $.Values.containerSecurityContext "imagePullPolicy" $.Values.image.pullPolicy }} + - {{- toYaml (merge $overwrites . $defaults) | nindent 10 }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 12 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- if .Values.kubeRBACProxy.image.sha }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}@sha256:{{ .Values.kubeRBACProxy.image.sha }}" + {{- else }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}" + {{- end }} + ports: + - containerPort: {{ .Values.service.port}} + name: {{ .Values.kubeRBACProxy.portName }} + {{- if .Values.kubeRBACProxy.enableHostPort }} + hostPort: {{ .Values.service.port }} + {{- end }} + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: + {{- toYaml .Values.kubeRBACProxy.resources | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + {{- with .Values.kubeRBACProxy.env }} + env: + {{- range $key, $value := $.Values.kubeRBACProxy.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: + {{ toYaml .Values.kubeRBACProxy.containerSecurityContext | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.restartPolicy }} + restartPolicy: {{ . }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + {{- if .Values.hostRootFsMount.enabled }} + - name: root + hostPath: + path: / + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- with $mount.type }} + type: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml new file mode 100644 index 0000000000..56b695203a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml @@ -0,0 +1,18 @@ +{{- if .Values.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +subsets: + - addresses: + {{- range .Values.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml new file mode 100644 index 0000000000..2b21b71062 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl . $ }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml new file mode 100644 index 0000000000..825722729d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml @@ -0,0 +1,23 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingress: + - ports: + - port: {{ .Values.service.port }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml new file mode 100644 index 0000000000..f88da6a34e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.podmonitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-node-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..ee5bbba4a5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml @@ -0,0 +1,14 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ include "prometheus-node-exporter.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..160f2bbf7a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ include "prometheus-node-exporter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml new file mode 100644 index 0000000000..f3b52e1120 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml @@ -0,0 +1,49 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.rbac.pspAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + - 'hostPath' + hostNetwork: true + hostIPC: false + hostPID: true + hostPorts: + - min: 0 + max: 65535 + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml new file mode 100644 index 0000000000..814e110337 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + namespace: {{ include "prometheus-node-exporter.namespace" . }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "prometheus-node-exporter.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "prometheus-node-exporter.fullname" . }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml new file mode 100644 index 0000000000..a065e46e39 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml @@ -0,0 +1,29 @@ +{{- if .Values.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.service.ipDualStack.ipFamilyPolicy }} +{{- end }} + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: {{ .Values.service.portName }} + selector: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..5c3348c09b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..6d6e440473 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.monitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + metricRelabelings: + {{- with .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..2c2705f872 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: node-exporter + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ . }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: {{ include "prometheus-node-exporter.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/values.yaml new file mode 100644 index 0000000000..b9f2f7ab87 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/values.yaml @@ -0,0 +1,530 @@ +# Default values for prometheus-node-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-node-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: v1.7.0 + pullPolicy: IfNotPresent + digest: "" + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +global: + cattle: + psp: + enable: true + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "docker.io" + +# Configure kube-rbac-proxy. When enabled, creates a kube-rbac-proxy to protect the node-exporter http endpoint. +# The requests are served through the same service but requests are HTTPS. +kubeRBACProxy: + enabled: false + ## Set environment variables as name/value pairs + env: {} + # VARIABLE: value + image: + registry: docker.io + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.15.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-proxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: {} + + # Specify the port used for the Node exporter container (upstream port) + port: 8100 + # Specify the name of the container port + portName: http + # Configure a hostPort. If true, hostPort will be enabled in the container and set to service.port. + enableHostPort: false + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +service: + enabled: true + type: ClusterIP + port: 9796 + targetPort: 9796 + nodePort: + portName: metrics + listenOnAllInterfaces: true + annotations: + prometheus.io/scrape: "true" + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + +# Set a NetworkPolicy with: +# ingress only on service.port +# no egress permitted +networkPolicy: + enabled: false + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + + jobLabel: "" + + # List of pod labels to add to node exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: [] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Node Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +# Specify the container restart policy passed to the Node Export container +# Possible Values: Always (default)|OnFailure|Never +restartPolicy: null + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + +containerSecurityContext: + readOnlyRootFilesystem: true + # capabilities: + # add: + # - SYS_TIME + +rbac: + ## If true, create & use RBAC resources + ## + create: true + pspAnnotations: {} + +# for deployments that have node_exporter deployed outside of the cluster, list +# their addresses here +endpoints: [] + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +# Mount the node's root file system (/) at /host/root in the container +hostRootFsMount: + enabled: true + # Defines how new mounts in existing mounts on the node or in the container + # are propagated to the container or node, respectively. Possible values are + # None, HostToContainer, and Bidirectional. If this field is omitted, then + # None is used. More information on: + # https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation + mountPropagation: HostToContainer + +# Mount the node's proc file system (/proc) at /host/proc in the container +hostProcFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +# Mount the node's sys file system (/sys) at /host/sys in the container +hostSysFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to node exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to node exporter pods +podLabels: {} + +# Annotations to be added to node exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-node-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: linux + # kubernetes.io/arch: amd64 + +# Specify grace period for graceful termination of pods. Defaults to 30 if null or not specified +terminationGracePeriodSeconds: null + +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + +# Enable or disable container termination message settings +# https://kubernetes.io/docs/tasks/debug/debug-application/determine-reason-pod-failure/ +terminationMessageParams: + enabled: false + # If enabled, specify the path for termination messages + terminationMessagePath: /dev/termination-log + # If enabled, specify the policy for termination messages + terminationMessagePolicy: File + + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.diskstats.ignored-devices=^(ram|loop|fd|(h|s|v)d[a-z]|nvme\\d+n\\d+p)\\d+$ +# - --collector.textfile.directory=/run/prometheus + +## Additional mounts from the host to node-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# https://kubernetes.io/docs/concepts/storage/volumes/#hostpath-volume-types +# type: "" (Default)|DirectoryOrCreate|Directory|FileOrCreate|File|Socket|CharDevice|BlockDevice +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file; fields image,imagePullPolicy,securityContext take default value from main container +## +sidecars: [] +# - name: nvidia-dcgm-exporter +# image: nvidia/dcgm-exporter:1.4.3 +# volumeMounts: +# - name: tmp +# mountPath: /tmp + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +# - name: collector-textfiles +# mountPath: /run/prometheus +# readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +# Enable vertical pod autoscaler support for prometheus-node-exporter +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# Extra manifests to deploy as an array +extraManifests: [] + # - | + # apiVersion: v1 + # kind: ConfigMap + # metadata: + # name: prometheus-extra + # data: + # extra-data: "value" + +# Override version of app, required if image.tag is defined and does not follow semver +version: "" diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/Chart.yaml new file mode 100644 index 0000000000..c50eeb0faf --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2ControllerManager +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/Chart.yaml new file mode 100644 index 0000000000..266f100ff9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Etcd +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/Chart.yaml new file mode 100644 index 0000000000..8978b71027 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2IngressNginx +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/Chart.yaml new file mode 100644 index 0000000000..a8d19e9326 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Proxy +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/Chart.yaml new file mode 100644 index 0000000000..bffedd10b1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Scheduler +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/Chart.yaml new file mode 100644 index 0000000000..76cc55be61 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeControllerManager +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/Chart.yaml new file mode 100644 index 0000000000..7f08ee0ba7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeEtcd +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/Chart.yaml new file mode 100644 index 0000000000..4fd9562394 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeIngressNginx +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/Chart.yaml new file mode 100644 index 0000000000..5ff503cf7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeProxy +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/Chart.yaml new file mode 100644 index 0000000000..7ff647b0c2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeScheduler +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/Chart.yaml new file mode 100644 index 0000000000..784bb0ec7e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +appVersion: 0.25.1 +description: A Helm chart for prometheus windows-exporter +home: https://github.com/prometheus-community/windows_exporter/ +keywords: +- windows-exporter +- windows +- prometheus +- exporter +maintainers: +- email: github@jkroepke.de + name: jkroepke +name: windowsExporter +sources: +- https://github.com/prometheus-community/windows_exporter/ +type: application +version: 0.3.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/README.md new file mode 100644 index 0000000000..1da1c64e12 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/README.md @@ -0,0 +1,42 @@ +# Prometheus `Windows Exporter` + +Prometheus exporter for hardware and OS metrics exposed by Windows kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a prometheus [`Windows Exporter`](http://github.com/prometheus-community/windows_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-windows-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-windows-exporter +``` diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 new file mode 100644 index 0000000000..9cbed7112d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 @@ -0,0 +1,31 @@ +$ErrorActionPreference = 'Continue' + +function CheckFirewallRuleError { + # We hit an error. This can happen for a number of reasons, including if the rule already exists + if ($error[0]) { + if (($error[0].Exception.NativeErrorCode) -and ($error[0].Exception.NativeErrorCode.ToString() -eq "AlreadyExists")) { + # Previous versions of monitoring may have already created this Firewall Rule + # Because of this, if the rule alreadys exists there is no need to delete and recreate it. + Write-Host "Detected Existing Firewall Rule, Nothing To Do" + } else { + Write-Host "Error Encountered Setting Up Required Firewall Rule" + $error[0].Exception + exit 1 + } + } +} + +Write-Host "Attempting To Configure Firewall Rules For Ports 9796, 10250" + +# This is the exact same firewall rule that has historically been created by rancher-wins +# https://github.com/rancher/wins/blob/91f670c47f19c6d9fe97d8f66a695d3081ad994f/pkg/apis/process_service_mgmt.go#L149 +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-9796 -Name rancher-wins-windows-exporter-TCP-9796 -Action Allow -Protocol TCP -LocalPort 9796 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Node Exporter Firewall Rule Successfully Created" + +# This rule is required in order to have the Rancher UI display node metrics in the 'Nodes' tab of the cluster explorer +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-10250 -Name rancher-wins-windows-exporter-TCP-10250 -Action Allow -Protocol TCP -LocalPort 10250 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Prometheus Metrics Firewall Rule Successfully Created" + +Write-Host "All Firewall Rules Successfully Configured" diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..c9a5d6db8c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/_helpers.tpl @@ -0,0 +1,216 @@ +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "prometheus-windows-exporter.fullname" -}} +{{ printf "%s-windows-exporter" .Release.Name }} +{{- end -}} + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "windowsExporter.renamedMetricsRelabeling" -}} +{{- range $original, $new := (include "windowsExporter.renamedMetrics" . | fromJson) -}} +- sourceLabels: [__name__] + regex: {{ $original }} + replacement: '{{ $new }}' + targetLabel: __name__ +{{ end -}} +{{- end -}} + +{{- define "windowsExporter.labels" -}} +k8s-app: {{ template "prometheus-windows-exporter.fullname" . }} +release: {{ .Release.Name }} +component: "windows-exporter" +provider: kubernetes +{{- end -}} + +{{- define "windowsExporter.renamedMetrics" -}} +{{- $renamed := dict -}} +{{/* v0.15.0 */}} +{{- $_ := set $renamed "windows_mssql_transactions_active_total" "windows_mssql_transactions_active" -}} +{{/* v0.16.0 */}} +{{- $_ := set $renamed "windows_adfs_ad_login_connection_failures" "windows_adfs_ad_login_connection_failures_total" -}} +{{- $_ := set $renamed "windows_adfs_certificate_authentications" "windows_adfs_certificate_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_device_authentications" "windows_adfs_device_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_extranet_account_lockouts" "windows_adfs_extranet_account_lockouts_total" -}} +{{- $_ := set $renamed "windows_adfs_federated_authentications" "windows_adfs_federated_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_passport_authentications" "windows_adfs_passport_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_failed" "windows_adfs_password_change_failed_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_succeeded" "windows_adfs_password_change_succeeded_total" -}} +{{- $_ := set $renamed "windows_adfs_token_requests" "windows_adfs_token_requests_total" -}} +{{- $_ := set $renamed "windows_adfs_windows_integrated_authentications" "windows_adfs_windows_integrated_authentications_total" -}} +{{- $_ := set $renamed "windows_net_packets_outbound_errors" "windows_net_packets_outbound_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_discarded" "windows_net_packets_received_discarded_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_errors" "windows_net_packets_received_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_total" "windows_net_packets_received_total_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_unknown" "windows_net_packets_received_unknown_total" -}} +{{- $_ := set $renamed "windows_dns_memory_used_bytes_total" "windows_dns_memory_used_bytes" -}} +{{- $renamed | toJson -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-windows-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-windows-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-windows-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-windows-exporter.name" . }} +{{ include "prometheus-windows-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-windows-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-windows-exporter.fullname" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-windows-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-windows-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-windows-exporter.image" -}} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-windows-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-windows-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-windows-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-windows-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/config.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/config.yaml new file mode 100644 index 0000000000..25f1fa69c2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + config.yml: | + {{- .Values.config | nindent 4 }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/daemonset.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/daemonset.yaml new file mode 100644 index 0000000000..be7feb3ed1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/daemonset.yaml @@ -0,0 +1,200 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "windowsExporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + initContainers: + - name: configure-firewall + image: {{ include "prometheus-windows-exporter.image" . }} + command: + - C:\WINDOWS\System32\WindowsPowerShell\v1.0\powershell.exe + args: ["-f", "scripts/configure-firewall.ps1"] + volumeMounts: + - mountPath: /scripts + name: exporter-scripts + {{- with .Values.extraInitContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-windows-exporter.fullname" . }} + containers: + - name: windows-exporter + image: {{ include "prometheus-windows-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --config.file=%CONTAINER_SANDBOX_MOUNT_POINT%/config.yml + - --collector.textfile.directories=%CONTAINER_SANDBOX_MOUNT_POINT% + - --web.listen-address=:{{ .Values.service.port }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + hostPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /config.yml + subPath: config.yml + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- if or .Values.sidecarVolumeMount .Values.sidecarHostVolumeMounts }} + volumeMounts: + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: exporter-scripts + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + - name: config + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml new file mode 100644 index 0000000000..bbb6c39340 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.podmonitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-windows-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-windows-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml new file mode 100644 index 0000000000..f514c8161a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{ (.Files.Glob "scripts/*").AsConfig | indent 2 }} + diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/service.yaml new file mode 100644 index 0000000000..267b796f63 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/service.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- if or .Values.prometheus.monitor.enabled .Values.prometheus.podMonitor.enabled }} + {{- with .Values.service.annotations }} + annotations: + {{- unset . "prometheus.io/scrape" | toYaml | nindent 4 }} + {{- end }} + {{- else }} + annotations: + prometheus.io/scrape: "true" + {{- with .Values.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port }} + protocol: TCP + appProtocol: http + name: {{ .Values.service.portName }} + selector: + {{- include "windowsExporter.labels" . | nindent 4 }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..14c1c46807 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-windows-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..2effc07758 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.monitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + metricRelabelings: +{{- include "windowsExporter.renamedMetricsRelabeling" . | nindent 6 -}} + - sourceLabels: [__name__] + regex: 'wmi_(.*)' + replacement: 'windows_$1' + targetLabel: __name__ + - sourceLabels: [volume, nic] + regex: (.*);(.*) + separator: '' + targetLabel: device + action: replace + replacement: $1$2 + - sourceLabels: [__name__] + regex: windows_cs_logical_processors + replacement: 'system' + targetLabel: mode + relabelings: + - separator: ':' + sourceLabels: + - __meta_kubernetes_pod_host_ip + - __meta_kubernetes_pod_container_port_number + targetLabel: instance +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/values.yaml new file mode 100644 index 0000000000..04569505d6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/values.yaml @@ -0,0 +1,366 @@ +# Default values for prometheus-windows-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-windows-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: "0.25.1" + pullPolicy: IfNotPresent + digest: "" + +config: |- + collectors: + enabled: '[defaults],tcp,memory,container' + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +global: + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + cattle: + systemDefaultRegistry: "" + +service: + type: ClusterIP + port: 9796 + nodePort: + portName: windows-metrics + annotations: {} + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: true + additionalLabels: {} + namespace: "" + + jobLabel: "component" + + # List of pod labels to add to windows exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: ["component"] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Windows Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m +# memory: 30Mi + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + +rbac: + ## If true, create & use RBAC resources + ## + create: true + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to windows exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to windows exporter pods +podLabels: {} + +# Annotations to be added to windows exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-windows-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: windows + # kubernetes.io/arch: amd64 + +tolerations: + - effect: NoSchedule + operator: Exists + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.service.services-where +# - "Name LIKE 'sql%'" + +## Additional mounts from the host to windows-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file +## +sidecars: [] +## - name: nvidia-dcgm-exporter +## image: nvidia/dcgm-exporter:1.4.3 + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +## - name: collector-textfiles +## mountPath: /run/prometheus +## readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/nginx.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/nginx.json new file mode 100644 index 0000000000..565352235a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/nginx.json @@ -0,0 +1,1445 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "$datasource", + "enable": true, + "expr": "sum(changes(nginx_ingress_controller_config_last_reload_successful_timestamp_seconds{instance!=\"unknown\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[30s])) by (controller_class)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Config Reloads", + "showIn": 0, + "step": "30s", + "tagKeys": "controller_class", + "tags": [], + "titleFormat": "Config Reloaded", + "type": "tags" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1534359654832, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 82, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",state=\"active\"}[2m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Connections", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Controller Success Rate (non-4|5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 81, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(irate(nginx_ingress_controller_success{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[1m])) * 60", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Config Reloads", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(nginx_ingress_controller_config_last_reload_successful{controller_pod=~\"$controller\",controller_namespace=~\"$namespace\"} == 0)", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Last Config Failed", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "200px", + "id": 86, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "network", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Request Volume", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Success Rate (non-4|5xx responses)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 77, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) ", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "", + "id": 79, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg (rate (nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) ", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "hideTimeOverride": false, + "id": 75, + "links": [], + "pageSize": 7, + "repeat": null, + "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Ingress", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ingress", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ops" + }, + { + "alias": "Errors", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "IN", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "OUT", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "Bps" + } + ], + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "E" + }, + { + "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "F" + }, + { + "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "G" + } + ], + "timeFrom": null, + "title": "Ingress Percentile Response Times and Transfer Rates", + "transform": "table", + "transparent": false, + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "height": "1024", + "id": 85, + "links": [], + "pageSize": 7, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "TTL", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0", + "691200" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "avg(nginx_ingress_controller_ssl_expire_time_seconds{kubernetes_pod_name=~\"$controller\",namespace=~\"$namespace\",ingress=~\"$ingress\"}) by (host) - time()", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ host }}", + "metric": "gke_letsencrypt_cert_expiration", + "refId": "A", + "step": 1 + } + ], + "title": "Ingress Certificate Expiry", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller Class", + "multi": false, + "name": "controller_class", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller", + "multi": false, + "name": "controller", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller_pod=~\"$controller\"}, ingress) ", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Ingress Controller", + "uid": "nginx", + "version": 1 +} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/request-handling-performance.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/request-handling-performance.json new file mode 100644 index 0000000000..156e33123d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/request-handling-performance.json @@ -0,0 +1,963 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": 9614, + "graphTooltip": 1, + "id": null, + "iteration": 1582146566338, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total time taken for nginx and upstream servers to process a request and send a response", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total request handling time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The time spent on receiving the response from the upstream server", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 94, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum by (path)(\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, its median upstream response time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n .5,\n sum by (le, path)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median upstream response time by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Percentage of 4xx and 5xx responses among all responses.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~ \"[4-5].*\"\n}[1m])) / sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error rate by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, the sum of upstream request time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_response_duration_seconds_sum{ingress =~ \"$ingress\"}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream time consumed by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum (\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~\"[4-5].*\",\n }[1m]\n )\n ) by(path, status)\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }} {{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate (\n nginx_ingress_controller_response_size_sum {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path) / sum (\n rate(\n nginx_ingress_controller_response_size_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "D" + }, + { + "expr": " sum (rate(nginx_ingress_controller_response_size_bucket{\n ingress =~ \"$ingress\",\n }[1m])) by (le)\n", + "hide": true, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average response size by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_sum {\n ingress =~ \"$ingress\",\n }[1m]\n)) / sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "average", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream service latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 22, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(nginx_ingress_controller_requests, ingress) ", + "hide": 0, + "includeAll": true, + "label": "Service Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests, ingress) ", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Request Handling Performance", + "uid": "4GFbkOsZk", + "version": 1 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json new file mode 100644 index 0000000000..1d4943501b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json @@ -0,0 +1,793 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m] ({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m])) by (instance)", + "interval": "", + "legendFormat": "Load[5m] ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m])) by (instance)", + "interval": "", + "legendFormat": "Load[1m] ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m])) by (instance)", + "interval": "", + "legendFormat": "Load[15m] ({{instance}})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) by (instance) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes) by (instance) ", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Read ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Write ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Errors ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Errors ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Dropped ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Dropped ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster (Nodes)", + "uid": "rancher-cluster-nodes-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster.json new file mode 100644 index 0000000000..24385a237a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster.json @@ -0,0 +1,776 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval]))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes)", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster", + "uid": "rancher-cluster-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundle.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundle.json new file mode 100644 index 0000000000..698f48aeed --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundle.json @@ -0,0 +1,246 @@ +{ + "description": "Bundle", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Bundle", + "uid": "fleet-bundle" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundledeployment.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundledeployment.json new file mode 100644 index 0000000000..c81f7a6212 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundledeployment.json @@ -0,0 +1,219 @@ +{ + "description": "BundleDeployment", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"}) / sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\"})" + } + ], + "title": "Ready BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundledeployment_state, cluster_namespace)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / BundleDeployment", + "uid": "fleet-bundledeployment" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/cluster.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/cluster.json new file mode 100644 index 0000000000..73bdea4834 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/cluster.json @@ -0,0 +1,484 @@ +{ + "description": "Cluster", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"}) / sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_desired_ready_git_repos, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_desired_ready_git_repos{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Cluster", + "uid": "fleet-cluster" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/clustergroup.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/clustergroup.json new file mode 100644 index 0000000000..ce3df87b21 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/clustergroup.json @@ -0,0 +1,468 @@ +{ + "description": "ClusterGroup", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"}) - sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})) / sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_group_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_group_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / ClusterGroup", + "uid": "fleet-cluster-group" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/controller-runtime.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/controller-runtime.json new file mode 100644 index 0000000000..23a81f2a8c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/controller-runtime.json @@ -0,0 +1,454 @@ +{ + "description": "Controller Runtime", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "controller_runtime_active_workers{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{controller}} {{instance}}" + } + ], + "title": "Number of Workers in Use", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Reconciliation Error Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Total Reconciliation Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "workqueue_depth{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "WorkQueue Depth", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P50", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P90", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P99", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_adds_total{job=\"$job\", namespace=\"$namespace\"}[2m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Add Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(workqueue_unfinished_work_seconds{job=\"$job\", namespace=\"$namespace\"}[5m])", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Unfinished Seconds", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 10, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 50th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 11, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 90th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 12, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 99th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 96 + }, + "id": 13, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_retries_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Retries Rate", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(controller_runtime_reconcile_total, namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "job", + "query": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"}, job)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Controller-Runtime", + "uid": "fleet-controller-runtime" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/gitrepo.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/gitrepo.json new file mode 100644 index 0000000000..1a50c2937d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/gitrepo.json @@ -0,0 +1,325 @@ +{ + "description": "GitRepo", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_gitrepo_desired_ready_clusters, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_gitrepo_desired_ready_clusters{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / GitRepo", + "uid": "fleet-gitrepo" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/home/rancher-default-home.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/home/rancher-default-home.json new file mode 100644 index 0000000000..3fce207561 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/home/rancher-default-home.json @@ -0,0 +1,1290 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "", + "type": "welcome" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[5m])))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "CPU Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 4 + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"})) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Memory Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 4 + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Disk Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 9 + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode!=\"idle\"}[5m]))", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 9 + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_cpu_cores{}) OR sum(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 9 + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 9 + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_memory_bytes{}) OR sum(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 9 + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) - sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 2051, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", mode=\"idle\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 2052, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "100 * (1- sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) by (instance) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) by (instance))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 2053, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(1 - ((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"} OR on() vector(0)))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0)))) * 100", + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "(1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "B" + }, + { + "expr": "(1 - (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) / sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "folderId": 0, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 18 + }, + "headings": true, + "id": 3, + "limit": 30, + "links": [], + "query": "", + "recent": true, + "search": true, + "starred": false, + "tags": [], + "title": "Dashboards", + "type": "dashlist" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 2055, + "options": { + "content": "## About Rancher Monitoring\n\nRancher Monitoring is a Helm chart developed by Rancher that is powered by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). It is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm chart maintained by the Prometheus community.\n\nBy default, the chart deploys Grafana alongside a set of Grafana dashboards curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project.\n\nFor more information on how Rancher Monitoring differs from [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack), please view the CHANGELOG.md of the rancher-monitoring chart located in the [rancher/charts](https://github.com/rancher/charts) repository.\n\nFor more information about how to configure Rancher Monitoring, please view the [Rancher docs](https://rancher.com/docs/rancher/v2.x/en/).\n\n", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "text" + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Home", + "uid": "rancher-home-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json new file mode 100644 index 0000000000..8af4b81ce0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json @@ -0,0 +1,687 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 32, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic In ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic Out ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes) by (instance)", + "interval": "", + "legendFormat": "DB Size ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Watch Streams ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Lease Watch Stream ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Committed ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Applied ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Failed ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending) by (instance)", + "interval": "", + "legendFormat": "Proposal Pending ({{instance}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Rate ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Failure Rate ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync ({{instance}})", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd (Nodes)", + "uid": "rancher-etcd-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd.json new file mode 100644 index 0000000000..0c058cafb9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd.json @@ -0,0 +1,669 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 33, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic In", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic Out", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes)", + "interval": "", + "legendFormat": "DB Size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Watch Streams", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Lease Watch Stream", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Committed", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Applied", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Failed", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending)", + "interval": "", + "legendFormat": "Proposal Pending", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Rate", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Failure Rate", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd", + "uid": "rancher-etcd-1", + "version": 4 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json new file mode 100644 index 0000000000..b31358eaaf --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json @@ -0,0 +1,527 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 30, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (instance, code)", + "interval": "", + "legendFormat": "{{code}}({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (instance, name)", + "interval": "", + "legendFormat": "Deployment Depth ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (instance, name)", + "interval": "", + "legendFormat": "Volumes Depth ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicaSet Depth ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (instance, name)", + "interval": "", + "legendFormat": "Service Depth ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (instance, name)", + "interval": "", + "legendFormat": "ServiceAccount Depth ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (instance, name)", + "interval": "", + "legendFormat": "Endpoint Depth ({{instance}})", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (instance, name)", + "interval": "", + "legendFormat": "DaemonSet Depth ({{instance}})", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (instance, name)", + "interval": "", + "legendFormat": "StatefulSet Depth ({{instance}})", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicationManager Depth ({{instance}})", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"}) by (instance)", + "interval": "", + "legendFormat": "Reading ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"}) by (instance)", + "interval": "", + "legendFormat": "Waiting ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"}) by (instance)", + "interval": "", + "legendFormat": "Writing ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Accepted ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Handled ({{instance}})", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components (Nodes)", + "uid": "rancher-k8s-components-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components.json new file mode 100644 index 0000000000..44cf97f9fd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components.json @@ -0,0 +1,519 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 31, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (code)", + "interval": "", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (name)", + "interval": "", + "legendFormat": "Deployment Depth", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (name)", + "interval": "", + "legendFormat": "Volumes Depth", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (name)", + "interval": "", + "legendFormat": "Replicaset Depth", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (name)", + "interval": "", + "legendFormat": "Service Depth", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (name)", + "interval": "", + "legendFormat": "ServiceAccount Depth", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (name)", + "interval": "", + "legendFormat": "Endpoint Depth", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (name)", + "interval": "", + "legendFormat": "DaemonSet Depth", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (name)", + "interval": "", + "legendFormat": "StatefulSet Depth", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (name)", + "interval": "", + "legendFormat": "ReplicationManager Depth", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"})", + "interval": "", + "legendFormat": "Reading", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"})", + "interval": "", + "legendFormat": "Waiting", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"})", + "interval": "", + "legendFormat": "Writing", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Accepted", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Handled", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components", + "uid": "rancher-k8s-components-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node-detail.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node-detail.json new file mode 100644 index 0000000000..920fb94cf7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node-detail.json @@ -0,0 +1,805 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{mode}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\"}[$__rate_interval])) by (mode)", + "interval": "", + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / (node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device))", + "interval": "", + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Read ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Write ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Errors ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Errors ({{device}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Dropped ({{device}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Dropped ({{device}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node (Detail)", + "uid": "rancher-node-detail-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node.json new file mode 100644 index 0000000000..367df3cc9d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node.json @@ -0,0 +1,792 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\", mode=\"idle\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / sum(node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node", + "uid": "rancher-node-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/performance/performance-debugging.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/performance/performance-debugging.json new file mode 100644 index 0000000000..454bc39390 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/performance/performance-debugging.json @@ -0,0 +1,1652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_sum[5m]))\n/\nsum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_count[5m])))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Average Execution Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1390", + "format": "short", + "label": "Execution Time in Seconds", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1391", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(steve_api_request_time_sum{resource!=\"subscribe\"}[5m]))\n/\nsum by (resource, method, code) (rate(steve_api_request_time_count{resource!=\"subscribe\"}[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Rancher API Average Request Times Over Last 5 Minutes (Top 20) (Subscribes Omitted)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:178", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:179", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "rate(steve_api_request_time_sum{resource=\"subscribe\"}[5m])\n/\nrate(steve_api_request_time_count{resource=\"subscribe\"}[5m])", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Subscribe Average Request Times Over Last 5 Minutes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:368", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:369", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,workqueue_depth)", + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Controller Work Queue Depth (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1553", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1554", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 16, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method, code) (steve_api_total_requests))", + "instant": false, + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Rancher Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:290", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:291", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method) (steve_api_total_requests{code!=\"200\",code!=\"201\"}))", + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Failed Rancher API Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:428", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:429", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_store_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_store_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Store Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:662", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:663", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 62 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_client_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_client_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Client Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1710", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1711", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,lasso_controller_total_cached_object)", + "interval": "", + "legendFormat": "{{kind}} {{version}} {{group}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cached Objects by GroupVersionKind (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:744", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:745", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 78 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Handler Executions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:824", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:825", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 86 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20, sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution{has_error=\"true\"}\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Handler Executions with Error (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1230", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1231", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 102 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution{has_error=\"true\"}[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 110 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,session_server_total_transmit_bytes)", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Data Transmitted by Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1953", + "format": "decbytes", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1954", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 118 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "session_server_total_transmit_error_bytes", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Errors for Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2045", + "format": "ms", + "label": "Error Data", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2046", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 126 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "session_server_total_add_websocket_session - (session_server_total_remove_websocket_session or (0 * session_server_total_add_websocket_session))", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Active Connections (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 134 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_remove_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Removed Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 142 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_add_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Added Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2117", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2118", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rancher Performance Debugging", + "uid": "tfrfU0a7k", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod-containers.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod-containers.json new file mode 100644 index 0000000000..cf78a2204c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod-containers.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "CFS throttled ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "System ({{container}})", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Total ({{container}})", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "User ({{container}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}) by (container)", + "interval": "", + "legendFormat": "({{container}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Dropped ({{container}})", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Errors ({{container}})", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Errors ({{container}})", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Dropped ({{container}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Write ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Read ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod (Containers)", + "uid": "rancher-pod-containers-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod.json new file mode 100644 index 0000000000..4859eccc74 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod", + "uid": "rancher-pod-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload-pods.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload-pods.json new file mode 100644 index 0000000000..92c0d24a6e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload-pods.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "CFS throttled ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "System ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Total ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "User ({{pod}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Dropped ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Errors ({{pod}})", + "refId": "D" + }, + { + "expr": "(sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Errors ({{pod}})", + "refId": "E" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Dropped ({{pod}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Write ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Read ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload (Pods)", + "uid": "rancher-workload-pods-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload.json new file mode 100644 index 0000000000..9f5317c2f0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum((sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum((sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(container_memory_working_set_bytes{namespace=~\"$namespace\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum((sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum((sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum((sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload", + "uid": "rancher-workload-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh b/charts/rancher-monitoring/103.2.2+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh new file mode 100644 index 0000000000..89431e7132 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e +set -x + +# node-exporter +kubectl delete daemonset -l app=prometheus-node-exporter,release=rancher-monitoring --ignore-not-found=true + +# prometheus-adapter +kubectl delete deployments -l app=prometheus-adapter,release=rancher-monitoring --ignore-not-found=true + +# kube-state-metrics +kubectl delete deployments -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true +kubectl delete statefulsets -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/NOTES.txt new file mode 100644 index 0000000000..371f3ae398 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "kube-prometheus-stack.namespace" . }} get pods -l "release={{ $.Release.Name }}" + +Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/_helpers.tpl new file mode 100644 index 0000000000..2827e81196 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/_helpers.tpl @@ -0,0 +1,467 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +{{/* +https://github.com/helm/helm/issues/4535#issuecomment-477778391 +Usage: {{ include "call-nested" (list . "SUBCHART_NAME" "TEMPLATE") }} +e.g. {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +*/}} +{{- define "call-nested" }} +{{- $dot := index . 0 }} +{{- $subchart := index . 1 | splitList "." }} +{{- $template := index . 2 }} +{{- $values := $dot.Values }} +{{- range $subchart }} +{{- $values = index $values . }} +{{- end }} +{{- include $template (dict "Chart" (dict "Name" (last $subchart)) "Values" $values "Release" $dot.Release "Capabilities" $dot.Capabilities) }} +{{- end }} + +# Special Exporters +{{- define "exporter.kubeEtcd.enabled" -}} +{{- if or .Values.kubeEtcd.enabled .Values.rkeEtcd.enabled .Values.kubeAdmEtcd.enabled .Values.rke2Etcd.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.enabled" -}} +{{- if or .Values.kubeControllerManager.enabled .Values.rkeControllerManager.enabled .Values.k3sServer.enabled .Values.kubeAdmControllerManager.enabled .Values.rke2ControllerManager.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.enabled" -}} +{{- if or .Values.kubeScheduler.enabled .Values.rkeScheduler.enabled .Values.k3sServer.enabled .Values.kubeAdmScheduler.enabled .Values.rke2Scheduler.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.enabled" -}} +{{- if or .Values.kubeProxy.enabled .Values.rkeProxy.enabled .Values.k3sServer.enabled .Values.kubeAdmProxy.enabled .Values.rke2Proxy.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.enabled" -}} +{{- if or .Values.kubelet.enabled .Values.hardenedKubelet.enabled .Values.k3sServer.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-controller-manager +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-scheduler +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-proxy +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kubelet +{{- end -}} +{{- end }} + +{{- define "kubelet.serviceMonitor.resourcePath" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if not (eq .Values.kubelet.serviceMonitor.resourcePath "/metrics/resource/v1alpha1") -}} +{{ .Values.kubelet.serviceMonitor.resourcePath }} +{{- else if semverCompare ">=1.20.0-0" $kubeTargetVersion -}} +/metrics/resource +{{- else -}} +/metrics/resource/v1alpha1 +{{- end -}} +{{- end }} + +{{- define "rancher.serviceMonitor.selector" -}} +{{- if .Values.rancherMonitoring.selector }} +{{ .Values.rancherMonitoring.selector | toYaml }} +{{- else }} +{{- $rancherDeployment := (lookup "apps/v1" "Deployment" "cattle-system" "rancher") }} +{{- if $rancherDeployment }} +matchLabels: + app: rancher + chart: {{ index $rancherDeployment.metadata.labels "chart" }} + release: rancher +{{- end }} +{{- end }} +{{- end }} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Prometheus Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "kube-prometheus-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "kube-prometheus-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 26 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Fullname suffixed with -operator */}} +{{/* Adding 9 to 26 truncation of kube-prometheus-stack.fullname */}} +{{- define "kube-prometheus-stack.operator.fullname" -}} +{{- if .Values.prometheusOperator.fullnameOverride -}} +{{- .Values.prometheusOperator.fullnameOverride | trunc 35 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-operator" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} +{{- end }} + +{{/* Prometheus custom resource instance name */}} +{{- define "kube-prometheus-stack.prometheus.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-prometheus" }} +{{- end }} +{{- end }} + +{{/* Prometheus apiVersion for networkpolicy */}} +{{- define "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} + +{{/* Alertmanager custom resource instance name */}} +{{- define "kube-prometheus-stack.alertmanager.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-alertmanager" -}} +{{- end }} +{{- end }} + +{{/* Fullname suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.fullname" -}} +{{- printf "%s-thanos-ruler" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} + +{{/* Shortened name suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.name" -}} +{{- default (printf "%s-thanos-ruler" (include "kube-prometheus-stack.name" .)) .Values.thanosRuler.name -}} +{{- end }} + + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "kube-prometheus-stack.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "kube-prometheus-stack.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }} +chart: {{ template "kube-prometheus-stack.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.operator.fullname" .) .Values.prometheusOperator.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (printf "%s-webhook" (include "kube-prometheus-stack.operator.fullname" .)) .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of prometheus service account to use */}} +{{- define "kube-prometheus-stack.prometheus.serviceAccountName" -}} +{{- if .Values.prometheus.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-prometheus") .Values.prometheus.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of alertmanager service account to use */}} +{{- define "kube-prometheus-stack.alertmanager.serviceAccountName" -}} +{{- if .Values.alertmanager.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-alertmanager") .Values.alertmanager.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.alertmanager.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of thanosRuler service account to use */}} +{{- define "kube-prometheus-stack.thanosRuler.serviceAccountName" -}} +{{- if .Values.thanosRuler.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.thanosRuler.name" .) .Values.thanosRuler.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.thanosRuler.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the grafana namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-grafana.namespace" -}} + {{- if .Values.grafana.namespaceOverride -}} + {{- .Values.grafana.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Allow kube-state-metrics job name to be overridden +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.name" -}} + {{- if index .Values "kube-state-metrics" "nameOverride" -}} + {{- index .Values "kube-state-metrics" "nameOverride" -}} + {{- else -}} + {{- print "kube-state-metrics" -}} + {{- end -}} +{{- end -}} + +{{/* +Use the kube-state-metrics namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.namespace" -}} + {{- if index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the prometheus-node-exporter namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-prometheus-node-exporter.namespace" -}} + {{- if index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Allow KubeVersion to be overridden. */}} +{{- define "kube-prometheus-stack.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{/* Get Ingress API Version */}} +{{- define "kube-prometheus-stack.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* Check Ingress stability */}} +{{- define "kube-prometheus-stack.ingress.isStable" -}} + {{- eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* Check Ingress supports pathType */}} +{{/* pathType was added to networking.k8s.io/v1beta1 in Kubernetes 1.18 */}} +{{- define "kube-prometheus-stack.ingress.supportsPathType" -}} + {{- or (eq (include "kube-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "kube-prometheus-stack.kubeVersion" .))) -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "kube-prometheus-stack.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} + {{- end -}} + +{{/* Get value based on current Kubernetes version */}} +{{- define "kube-prometheus-stack.kubeVersionDefaultValue" -}} + {{- $values := index . 0 -}} + {{- $kubeVersion := index . 1 -}} + {{- $old := index . 2 -}} + {{- $new := index . 3 -}} + {{- $default := index . 4 -}} + {{- if kindIs "invalid" $default -}} + {{- if semverCompare $kubeVersion (include "kube-prometheus-stack.kubeVersion" $values) -}} + {{- print $new -}} + {{- else -}} + {{- print $old -}} + {{- end -}} + {{- else -}} + {{- print $default }} + {{- end -}} +{{- end -}} + +{{/* Get value for kube-controller-manager depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeControllerManager.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.22-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Get value for kube-scheduler depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeScheduler.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.23-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +To help compatibility with other charts which use global.imagePullSecrets. +Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). +global: + imagePullSecrets: + - name: pullSecret1 + - name: pullSecret2 + +or + +global: + imagePullSecrets: + - pullSecret1 + - pullSecret2 +*/}} +{{- define "kube-prometheus-stack.imagePullSecrets" -}} +{{- range .Values.global.imagePullSecrets }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{- define "kube-prometheus-stack.operator.admission-webhook.dnsNames" }} +{{- $fullname := include "kube-prometheus-stack.operator.fullname" . }} +{{- $namespace := include "kube-prometheus-stack.namespace" . }} +{{- $fullname }} +{{ $fullname }}.{{ $namespace }}.svc +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +{{ $fullname }}-webhook +{{ $fullname }}-webhook.{{ $namespace }}.svc +{{- end }} +{{- end }} + +{{- define "rke2-ingress-nginx.namespace" -}} + {{- if .Values.rke2IngressNginx.namespaceOverride -}} + {{- .Values.rke2IngressNginx.namespaceOverride -}} + {{- else -}} + {{- print "kube-system" -}} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/alertmanager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/alertmanager.yaml new file mode 100644 index 0000000000..19044054ac --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/alertmanager.yaml @@ -0,0 +1,191 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.annotations }} + annotations: +{{ toYaml .Values.alertmanager.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.alertmanagerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.alertmanager.alertmanagerSpec.image.registry }} + {{- if and .Values.alertmanager.alertmanagerSpec.image.tag .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}" + {{- end }} + version: {{ .Values.alertmanager.alertmanagerSpec.image.tag }} + {{- if .Values.alertmanager.alertmanagerSpec.image.sha }} + sha: {{ .Values.alertmanager.alertmanagerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.alertmanager.alertmanagerSpec.replicas }} + listenLocal: {{ .Values.alertmanager.alertmanagerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.alertmanager.alertmanagerSpec.automountServiceAccountToken }} +{{- if .Values.alertmanager.alertmanagerSpec.externalUrl }} + externalUrl: "{{ tpl .Values.alertmanager.alertmanagerSpec.externalUrl . }}" +{{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.alertmanager.ingress.hosts 0) . }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ .Values.namespaceOverride }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.nodeSelector }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.alertmanager.alertmanagerSpec.paused }} + logFormat: {{ .Values.alertmanager.alertmanagerSpec.logFormat | quote }} + logLevel: {{ .Values.alertmanager.alertmanagerSpec.logLevel | quote }} + retention: {{ .Values.alertmanager.alertmanagerSpec.retention | quote }} +{{- if .Values.alertmanager.alertmanagerSpec.secrets }} + secrets: +{{ toYaml .Values.alertmanager.alertmanagerSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configSecret }} + configSecret: {{ .Values.alertmanager.alertmanagerSpec.configSecret }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configMaps }} + configMaps: +{{ toYaml .Values.alertmanager.alertmanagerSpec.configMaps | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector }} + alertmanagerConfigSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector | indent 4) . }} +{{ else }} + alertmanagerConfigSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector }} + alertmanagerConfigNamespaceSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector | indent 4) . }} +{{ else }} + alertmanagerConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.web }} + web: +{{ toYaml .Values.alertmanager.alertmanagerSpec.web | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration }} + alertmanagerConfiguration: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy }} + alertmanagerConfigMatcherStrategy: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.resources }} + resources: +{{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + routePrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.alertmanagerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.storage }} + storage: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.storage | indent 4) . }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.alertmanager.alertmanagerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.alertmanager.alertmanagerSpec.podAntiAffinity .Values.alertmanager.alertmanagerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.affinity }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.tolerations }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.containers }} + containers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.initContainers }} + initContainers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.priorityClassName }} + priorityClassName: {{.Values.alertmanager.alertmanagerSpec.priorityClassName }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.additionalPeers }} + additionalPeers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.additionalPeers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumes }} + volumes: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.alertmanager.alertmanagerSpec.portName }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} + clusterAdvertiseAddress: {{ .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} + clusterGossipInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} + clusterPeerTimeout: {{ .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} + clusterPushpullInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} + forceEnableClusterMode: {{ .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.minReadySeconds }} + minReadySeconds: {{ .Values.alertmanager.alertmanagerSpec.minReadySeconds }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/extrasecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/extrasecret.yaml new file mode 100644 index 0000000000..ecd8f47021 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.alertmanager.extraSecret.data -}} +{{- $secretName := printf "alertmanager-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.alertmanager.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.extraSecret.annotations }} + annotations: +{{ toYaml .Values.alertmanager.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.alertmanager.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingress.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingress.yaml new file mode 100644 index 0000000000..be9f5aa279 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingress.yaml @@ -0,0 +1,78 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} +{{- $pathType := .Values.alertmanager.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $backendServiceName := .Values.alertmanager.ingress.serviceName | default (printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager") }} +{{- $servicePort := .Values.alertmanager.ingress.servicePort | default .Values.alertmanager.service.port -}} +{{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} +{{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.alertmanager.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.alertmanager.ingress.labels }} +{{ toYaml .Values.alertmanager.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.alertmanager.ingress.ingressClassName }} + ingressClassName: {{ .Values.alertmanager.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.alertmanager.ingress.hosts }} + {{- range $host := .Values.alertmanager.ingress.hosts }} + - host: {{ tpl $host $ | quote }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.alertmanager.ingress.tls }} + tls: +{{ tpl (toYaml .Values.alertmanager.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingressperreplica.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingressperreplica.yaml new file mode 100644 index 0000000000..b2e00a4162 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled .Values.alertmanager.ingressPerReplica.enabled }} +{{- $pathType := .Values.alertmanager.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $servicePort := .Values.alertmanager.service.port -}} +{{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml new file mode 100644 index 0000000000..b183403125 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.alertmanager.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.alertmanager.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-role.yaml new file mode 100644 index 0000000000..8810e93ded --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-rolebinding.yaml new file mode 100644 index 0000000000..794f4ad178 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp.yaml new file mode 100644 index 0000000000..07b616b5cb --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/secret.yaml new file mode 100644 index 0000000000..d2fe84a7bf --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/secret.yaml @@ -0,0 +1,35 @@ +{{- if and (.Values.alertmanager.enabled) (not .Values.alertmanager.alertmanagerSpec.useExistingSecret) }} +{{/* This file is applied when the operation is helm install and the target secret does not exist. */}} +{{- $secretName := (printf "alertmanager-%s" (include "kube-prometheus-stack.alertmanager.crname" .)) }} +{{- if or (not (lookup "v1" "Secret" (include "kube-prometheus-stack.namespace" .) $secretName)) (eq .Values.alertmanager.secret.recreateIfExists true) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/resource-policy": keep +{{- if .Values.alertmanager.secret.annotations }} +{{ toYaml .Values.alertmanager.secret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if .Values.alertmanager.tplConfig }} +{{- if .Values.alertmanager.stringConfig }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.stringConfig) . | b64enc | quote }} +{{- else if eq (typeOf .Values.alertmanager.config) "string" }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.config) . | b64enc | quote }} +{{- else }} + alertmanager.yaml: {{ tpl (toYaml .Values.alertmanager.config) . | b64enc | quote }} +{{- end }} +{{- else }} + alertmanager.yaml: {{ toYaml .Values.alertmanager.config | b64enc | quote }} +{{- end }} +{{- range $key, $val := .Values.alertmanager.templateFiles }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/service.yaml new file mode 100644 index 0000000000..373de328a5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/service.yaml @@ -0,0 +1,68 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.alertmanager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + self-monitor: {{ .Values.alertmanager.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.service.labels }} +{{ toYaml .Values.alertmanager.service.labels | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.annotations }} + annotations: +{{ toYaml .Values.alertmanager.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.service.clusterIP }} + clusterIP: {{ .Values.alertmanager.service.clusterIP }} +{{- end }} +{{- if .Values.alertmanager.service.externalIPs }} + externalIPs: +{{ toYaml .Values.alertmanager.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.alertmanager.service.loadBalancerIP }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.alertmanager.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.alertmanager.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq .Values.alertmanager.service.type "NodePort" }} + nodePort: {{ .Values.alertmanager.service.nodePort }} + {{- end }} + port: {{ .Values.alertmanager.service.port }} + targetPort: {{ .Values.alertmanager.service.targetPort }} + protocol: TCP + - name: reloader-web + {{- if semverCompare ">=1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: 8080 + targetPort: reloader-web +{{- if .Values.alertmanager.service.additionalPorts }} +{{ toYaml .Values.alertmanager.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- if .Values.alertmanager.service.sessionAffinity }} + sessionAffinity: {{ .Values.alertmanager.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.alertmanager.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.alertmanager.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.alertmanager.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceaccount.yaml new file mode 100644 index 0000000000..745ced8bde --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.alertmanager.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.alertmanager.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2}} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/servicemonitor.yaml new file mode 100644 index 0000000000..6233690019 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/servicemonitor.yaml @@ -0,0 +1,84 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.alertmanager.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.alertmanager.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + enableHttp2: {{ .Values.alertmanager.serviceMonitor.enableHttp2 }} + {{- if .Values.alertmanager.serviceMonitor.interval }} + interval: {{ .Values.alertmanager.serviceMonitor.interval }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.alertmanager.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.scheme }} + scheme: {{ .Values.alertmanager.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.alertmanager.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.alertmanager.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.alertmanager.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.alertmanager.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.alertmanager.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.alertmanager.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.alertmanager.serviceMonitor.interval .interval }} + interval: {{ default $.Values.alertmanager.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.alertmanager.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.alertmanager.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceperreplica.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceperreplica.yaml new file mode 100644 index 0000000000..75a13bdf97 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceperreplica.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $serviceValues := .Values.alertmanager.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" $ }} + statefulset.kubernetes.io/pod-name: alertmanager-{{ include "kube-prometheus-stack.alertmanager.crname" $ }}-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/service.yaml new file mode 100644 index 0000000000..b8618f7558 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/service.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + jobLabel: coredns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.coreDns.serviceMonitor.port }} + port: {{ .Values.coreDns.service.port }} + protocol: TCP + targetPort: {{ .Values.coreDns.service.targetPort }} + selector: + {{- if .Values.coreDns.service.selector }} +{{ toYaml .Values.coreDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml new file mode 100644 index 0000000000..dc15a06937 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + {{- with .Values.coreDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.coreDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.coreDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.coreDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.coreDns.serviceMonitor.port }} + {{- if .Values.coreDns.serviceMonitor.interval}} + interval: {{ .Values.coreDns.serviceMonitor.interval }} + {{- end }} + {{- if .Values.coreDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.coreDns.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.coreDns.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.coreDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.coreDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml new file mode 100644 index 0000000000..66e777632e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.kubeApiServer.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-apiserver + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: default + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-apiserver + {{- with .Values.kubeApiServer.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubeApiServer.serviceMonitor | nindent 2 }} + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeApiServer.serviceMonitor.interval }} + interval: {{ .Values.kubeApiServer.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubeApiServer.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeApiServer.serviceMonitor.proxyUrl }} + {{- end }} + port: https + scheme: https + metricRelabelings: + {{- if .Values.kubeApiServer.serviceMonitor.metricRelabelings }} +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeApiServer.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.relabelings | indent 6) . }} +{{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + serverName: {{ .Values.kubeApiServer.tlsConfig.serverName }} + insecureSkipVerify: {{ .Values.kubeApiServer.tlsConfig.insecureSkipVerify }} + jobLabel: {{ .Values.kubeApiServer.serviceMonitor.jobLabel }} + namespaceSelector: + matchNames: + - default + selector: +{{ toYaml .Values.kubeApiServer.serviceMonitor.selector | indent 4 }} +{{- end}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml new file mode 100644 index 0000000000..6a6afa6412 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + k8s-app: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeControllerManager.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/service.yaml new file mode 100644 index 0000000000..43b1a976d5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + jobLabel: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.targetPort) }} +{{- if .Values.kubeControllerManager.endpoints }}{{- else }} + selector: + {{- if .Values.kubeControllerManager.service.selector }} +{{ toYaml .Values.kubeControllerManager.service.selector | indent 4 }} + {{- else}} + component: kube-controller-manager + {{- end}} +{{- end }} + type: ClusterIP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml new file mode 100644 index 0000000000..7ed3baa65f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + {{- with .Values.kubeControllerManager.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeControllerManager.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeControllerManager.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeControllerManager.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- if .Values.kubeControllerManager.serviceMonitor.interval }} + interval: {{ .Values.kubeControllerManager.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeControllerManager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeControllerManager.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . false true .Values.kubeControllerManager.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . nil true .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeControllerManager.serviceMonitor.serverName }} + serverName: {{ .Values.kubeControllerManager.serviceMonitor.serverName }} + {{- end }} + {{- end }} + metricRelabelings: + {{- if.Values.kubeControllerManager.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeControllerManager.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/service.yaml new file mode 100644 index 0000000000..81b2c9930c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/service.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + jobLabel: kube-dns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics-dnsmasq + port: {{ .Values.kubeDns.service.dnsmasq.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.dnsmasq.targetPort }} + - name: http-metrics-skydns + port: {{ .Values.kubeDns.service.skydns.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.skydns.targetPort }} + selector: + {{- if .Values.kubeDns.service.selector }} +{{ toYaml .Values.kubeDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml new file mode 100644 index 0000000000..9fa41b575f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + {{- with .Values.kubeDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics-dnsmasq + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeDns.serviceMonitor.proxyUrl}} + {{- end }} + metricRelabelings: + {{- if .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeDns.serviceMonitor.dnsmasqRelabelings }} + relabelings: +{{ toYaml .Values.kubeDns.serviceMonitor.dnsmasqRelabelings | indent 4 }} +{{- end }} + - port: http-metrics-skydns + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubeDns.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml new file mode 100644 index 0000000000..e366447577 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + k8s-app: etcd-server +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeEtcd.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/service.yaml new file mode 100644 index 0000000000..d07d4f35e3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + jobLabel: kube-etcd +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeEtcd.service.targetPort }} +{{- if .Values.kubeEtcd.endpoints }}{{- else }} + selector: + {{- if .Values.kubeEtcd.service.selector }} +{{ toYaml .Values.kubeEtcd.service.selector | indent 4 }} + {{- else}} + component: etcd + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml new file mode 100644 index 0000000000..26fdbdbed3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + {{- with .Values.kubeEtcd.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeEtcd.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeEtcd.serviceMonitor | nindent 4 }} + selector: + {{- if .Values.kubeEtcd.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeEtcd.serviceMonitor.port }} + {{- if .Values.kubeEtcd.serviceMonitor.interval }} + interval: {{ .Values.kubeEtcd.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeEtcd.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeEtcd.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq .Values.kubeEtcd.serviceMonitor.scheme "https" }} + scheme: https + tlsConfig: + {{- if .Values.kubeEtcd.serviceMonitor.serverName }} + serverName: {{ .Values.kubeEtcd.serviceMonitor.serverName }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.caFile }} + caFile: {{ .Values.kubeEtcd.serviceMonitor.caFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.certFile }} + certFile: {{ .Values.kubeEtcd.serviceMonitor.certFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.keyFile }} + keyFile: {{ .Values.kubeEtcd.serviceMonitor.keyFile }} + {{- end}} + insecureSkipVerify: {{ .Values.kubeEtcd.serviceMonitor.insecureSkipVerify }} + {{- end }} + metricRelabelings: + {{- if .Values.kubeEtcd.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeEtcd.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml new file mode 100644 index 0000000000..8613e62425 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + k8s-app: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeProxy.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/service.yaml new file mode 100644 index 0000000000..8ccb2210d7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + jobLabel: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeProxy.service.targetPort }} +{{- if .Values.kubeProxy.endpoints }}{{- else }} + selector: + {{- if .Values.kubeProxy.service.selector }} +{{ toYaml .Values.kubeProxy.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-proxy + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml new file mode 100644 index 0000000000..24b0ab2001 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + {{- with .Values.kubeProxy.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeProxy.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeProxy.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeProxy.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeProxy.serviceMonitor.port }} + {{- if .Values.kubeProxy.serviceMonitor.interval }} + interval: {{ .Values.kubeProxy.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeProxy.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeProxy.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.kubeProxy.serviceMonitor.https }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- end}} + metricRelabelings: + {{- if .Values.kubeProxy.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeProxy.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeProxy.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml new file mode 100644 index 0000000000..6236b42f10 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + k8s-app: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeScheduler.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/service.yaml new file mode 100644 index 0000000000..90b3a800a4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + jobLabel: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.targetPort) }} +{{- if .Values.kubeScheduler.endpoints }}{{- else }} + selector: + {{- if .Values.kubeScheduler.service.selector }} +{{ toYaml .Values.kubeScheduler.service.selector | indent 4 }} + {{- else}} + component: kube-scheduler + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml new file mode 100644 index 0000000000..b17c4f1d47 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + {{- with .Values.kubeScheduler.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeScheduler.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeScheduler.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeScheduler.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- if .Values.kubeScheduler.serviceMonitor.interval }} + interval: {{ .Values.kubeScheduler.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeScheduler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeScheduler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . false true .Values.kubeScheduler.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . nil true .Values.kubeScheduler.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeScheduler.serviceMonitor.serverName }} + serverName: {{ .Values.kubeScheduler.serviceMonitor.serverName }} + {{- end}} + {{- end}} + metricRelabelings: + {{- if .Values.kubeScheduler.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeScheduler.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml new file mode 100644 index 0000000000..9211b3d771 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml @@ -0,0 +1,7 @@ +{{- if .Values.kubeStateMetrics.enabled }} +{{- if not (kindIs "invalid" .Values.kubeStateMetrics.serviceMonitor) }} +{{- if .Values.kubeStateMetrics.serviceMonitor.namespaceOverride }} +{{- fail "kubeStateMetrics.serviceMonitor.namespaceOverride was removed. Please use kube-state-metrics.namespaceOverride instead." }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml new file mode 100644 index 0000000000..f570fbfdbc --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml @@ -0,0 +1,246 @@ +{{- if (and (not .Values.kubelet.enabled) .Values.hardenedKubelet.enabled) }} +{{ required "Cannot set .Values.hardenedKubelet.enabled=true when .Values.kubelet.enabled=false" "" }} +{{- end }} +{{- if (and .Values.kubelet.enabled .Values.kubernetesServiceMonitors.enabled (not .Values.hardenedKubelet.enabled) (not .Values.k3sServer.enabled)) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kubelet + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: {{ .Values.kubelet.namespace }} + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kubelet + {{- with .Values.kubelet.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubelet.serviceMonitor | nindent 2 }} + {{- with .Values.kubelet.serviceMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + {{- if .Values.kubelet.serviceMonitor.https }} + - port: https-metrics + scheme: https + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + metricRelabelings: + {{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: https-metrics + scheme: https + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: https-metrics + scheme: https + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: https-metrics + scheme: https + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} + {{- else }} + - port: http-metrics + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: http-metrics + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: http-metrics + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: http-metrics + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- end }} + {{- end }} + jobLabel: k8s-app + namespaceSelector: + matchNames: + - {{ .Values.kubelet.namespace }} + selector: + matchLabels: + app.kubernetes.io/name: kubelet + k8s-app: kubelet +{{- end}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/node-exporter/validate.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/node-exporter/validate.yaml new file mode 100644 index 0000000000..bdc73d6165 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/node-exporter/validate.yaml @@ -0,0 +1,3 @@ +{{- if (and (not .Values.nodeExporter.enabled) .Values.hardenedNodeExporter.enabled) }} +{{ required "Cannot set .Values.hardenedNodeExporter.enabled=true when .Values.nodeExporter.enabled=false" "" }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/extra-objects.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/extra-objects.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/extra-objects.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmap-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmap-dashboards.yaml new file mode 100644 index 0000000000..e719009ffe --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmap-dashboards.yaml @@ -0,0 +1,24 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled) .Values.grafana.forceDeployDashboards }} +{{- $files := .Files.Glob "dashboards-1.14/*.json" }} +{{- if $files }} +apiVersion: v1 +kind: ConfigMapList +items: +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 6 }} + data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmaps-datasources.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmaps-datasources.yaml new file mode 100644 index 0000000000..718020d4f6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmaps-datasources.yaml @@ -0,0 +1,81 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled) .Values.grafana.forceDeployDatasources }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-grafana-datasource + namespace: {{ default .Values.grafana.sidecar.datasources.searchNamespace (include "kube-prometheus-stack.namespace" .) }} +{{- if .Values.grafana.sidecar.datasources.annotations }} + annotations: + {{- toYaml .Values.grafana.sidecar.datasources.annotations | nindent 4 }} +{{- end }} + labels: + {{ $.Values.grafana.sidecar.datasources.label }}: {{ $.Values.grafana.sidecar.datasources.labelValue | quote }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + datasource.yaml: |- + apiVersion: 1 +{{- if .Values.grafana.deleteDatasources }} + deleteDatasources: +{{ tpl (toYaml .Values.grafana.deleteDatasources | indent 6) . }} +{{- end }} + datasources: +{{- $scrapeInterval := .Values.grafana.sidecar.datasources.defaultDatasourceScrapeInterval | default .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} +{{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} + - name: Prometheus + type: prometheus + uid: {{ .Values.grafana.sidecar.datasources.uid }} + {{- if .Values.grafana.sidecar.datasources.url }} + url: {{ .Values.grafana.sidecar.datasources.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + {{- end }} + access: proxy + isDefault: {{ .Values.grafana.sidecar.datasources.isDefaultDatasource }} + jsonData: + httpMethod: {{ .Values.grafana.sidecar.datasources.httpMethod }} + timeInterval: {{ $scrapeInterval }} + {{- if .Values.grafana.sidecar.datasources.timeout }} + timeout: {{ .Values.grafana.sidecar.datasources.timeout }} + {{- end }} +{{- if .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.createPrometheusReplicasDatasources }} +{{- range until (int .Values.prometheus.prometheusSpec.replicas) }} + - name: Prometheus-{{ . }} + type: prometheus + uid: {{ $.Values.grafana.sidecar.datasources.uid }}-replica-{{ . }} + url: http://prometheus-{{ template "kube-prometheus-stack.prometheus.crname" $ }}-{{ . }}.prometheus-operated:9090/{{ trimPrefix "/" $.Values.prometheus.prometheusSpec.routePrefix }} + access: proxy + isDefault: false + jsonData: + timeInterval: {{ $scrapeInterval }} +{{- if $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.alertmanager.enabled }} + - name: Alertmanager + type: alertmanager + uid: {{ .Values.grafana.sidecar.datasources.alertmanager.uid }} + {{- if .Values.grafana.sidecar.datasources.alertmanager.url }} + url: {{ .Values.grafana.sidecar.datasources.alertmanager.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }}/{{ trimPrefix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }} + {{- end }} + access: proxy + jsonData: + handleGrafanaManagedAlerts: {{ .Values.grafana.sidecar.datasources.alertmanager.handleGrafanaManagedAlerts }} + implementation: {{ .Values.grafana.sidecar.datasources.alertmanager.implementation }} +{{- end }} +{{- end }} +{{- if .Values.grafana.additionalDataSources }} +{{ tpl (toYaml .Values.grafana.additionalDataSources | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml new file mode 100644 index 0000000000..dfc26d7ecd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml @@ -0,0 +1,616 @@ +{{- /* +Generated from 'alertmanager-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "alertmanager-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + alertmanager-overview.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "current set of alerts stored in the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid alerts received by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Received", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Invalid", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts receive rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alerts", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Failed", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notifications Send Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "latency of notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} 99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Median", + "refId": "B" + }, + { + "expr": "sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notification Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Notifications", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "alertmanager-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "service", + "multi": false, + "name": "service", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, service)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "integration", + "options": [ + + ], + "query": "label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Alertmanager / Overview", + "uid": "alertmanager-overview", + "version": 0 + } +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml new file mode 100644 index 0000000000..bd1048b567 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml @@ -0,0 +1,1772 @@ +{{- /* +Generated from 'apiserver' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeApiServer.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "apiserver" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + apiserver.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "datasource": null, + "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "mode": "markdown", + "span": 12, + "title": "Notice", + "type": "text" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Availability (30d) > 99.000%", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "description": "How much error budget is left looking at our 0.990% availability guarantees?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "errorbudget", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ErrorBudget (30d) > 99.000%", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Read Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many read requests (LIST,GET) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Write Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / API server", + "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml new file mode 100644 index 0000000000..f4be0bbd45 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -0,0 +1,1882 @@ +{{- /* +Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "cluster-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + cluster-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "namespace", + "value": "namespace" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth History", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "What is TCP Retransmit?", + "url": "https://accedian.com/enterprises/blog/network-packet-loss-retransmissions-and-duplicate-acknowledgements/" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP Retransmits out of all sent segments", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Why monitor SYN retransmits?", + "url": "https://github.com/prometheus/node_exporter/issues/1023#issuecomment-408128365" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP SYN Retransmits out of all retransmits", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Cluster", + "uid": "ff635a025bcfea7bc3dd4f508990a3e9", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml new file mode 100644 index 0000000000..8d420d7a4f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -0,0 +1,1196 @@ +{{- /* +Generated from 'controller-manager' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "controller-manager" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + controller-manager.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Controller Manager", + "uid": "72e0e05bef5099e5f049b05fdc429ed4", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml new file mode 100644 index 0000000000..0eeedc6299 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml @@ -0,0 +1,1229 @@ +{{- /* +Generated from 'etcd' from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "etcd" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + etcd.json: |- + { + "annotations": { + "list": [] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 42, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"$cluster\"}[$__rate_interval])))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer round trip time", + "metric": "etcd_network_peer_round_trip_time_seconds_bucket", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Peer round trip time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:925", + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:926", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + "etcd-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": {{ if (or .Values.grafana.sidecar.dashboards.multicluster.global.enabled .Values.grafana.sidecar.dashboards.multicluster.etcd.enabled) }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 2, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "etcd", + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", + "version": 215 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml new file mode 100644 index 0000000000..d2609140cf --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml @@ -0,0 +1,635 @@ +{{- /* +Generated from 'grafana-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "grafana-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + grafana-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + + ], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3085, + "iteration": 1631554945276, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Firing Alerts", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dashboards", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "displayMode": "auto" + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "showHeader": true + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_build_info{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Build Info", + "transformations": [ + { + "id": "labelsToFields", + "options": { + + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "branch": true, + "container": true, + "goversion": true, + "namespace": true, + "pod": true, + "revision": true + }, + "indexByName": { + "Time": 7, + "Value": 11, + "branch": 4, + "container": 8, + "edition": 2, + "goversion": 6, + "instance": 1, + "job": 0, + "namespace": 9, + "pod": 10, + "revision": 5, + "version": 3 + }, + "renameByName": { + + } + } + } + ], + "type": "table" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ", + "interval": "", + "legendFormat": "{{`{{`}}status_code{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "RPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:157", + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:210", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:211", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 30, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": [ + "default/grafana" + ], + "value": [ + "default/grafana" + ] + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, job)", + "refId": "Billing Admin-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "instance", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, instance)", + "refId": "Billing Admin-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Grafana Overview", + "uid": "6be0s85Mk", + "version": 2 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml new file mode 100644 index 0000000000..7ecca76f23 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml @@ -0,0 +1,1534 @@ +{{- /* +Generated from 'k8s-coredns' from ../files/dashboards/k8s-coredns.json +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.coreDns.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-coredns" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-coredns.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for the CoreDNS DNS server with updated metrics for version 1.7.0+. Based on the CoreDNS dashboard by buhay.", + "editable": true, + "gnetId": 12539, + "graphTooltip": 0, + "iteration": 1603798405693, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "CoreDNS.io", + "type": "link", + "url": "https://coredns.io" + } + ], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (total)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + }, + { + "alias": "other", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type) or \nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by qtype)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{zone}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by zone)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_do_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_do_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DO", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (DO bit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 2 + }, + { + "alias": "tcp:99 ", + "yaxis": 2 + }, + { + "alias": "tcp:50", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 1 + }, + { + "alias": "tcp:99 ", + "yaxis": 1 + }, + { + "alias": "tcp:50", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size,tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode) or\nsum(rate(coredns_dns_responses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{rcode}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (by rcode)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le, job))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (duration)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + }, + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{job=\"coredns\",instance=~\"$instance\"}) by (type) or\nsum(coredns_cache_entries{job=\"coredns\",instance=~\"$instance\"}) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (size)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "misses", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "hits:{{"{{type}}"}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_cache_misses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "misses", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (hitrate)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "dns", + "coredns" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(up{job=\"coredns\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=\"coredns\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "CoreDNS", + "uid": "vkQ0UHxik", + "version": 2 + } +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml new file mode 100644 index 0000000000..93ee57db93 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -0,0 +1,3088 @@ +{{- /* +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Namespace", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 20, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 21, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 22, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml new file mode 100644 index 0000000000..9c295831a5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-multicluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-multicluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-multicluster.json: |- + {{`{"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster:node_cpu:ratio_rate5m","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Cluster","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Multi-Cluster","uid":"b59e6c9f2fcbe2e16d77fc492374cc4f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml new file mode 100644 index 0000000000..1c32c9c02e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -0,0 +1,2797 @@ +{{- /* +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml new file mode 100644 index 0000000000..e60a42d747 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -0,0 +1,1026 @@ +{{- /* +Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-node" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-node.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml new file mode 100644 index 0000000000..80fab51c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -0,0 +1,2469 @@ +{{- /* +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.25, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Throttling", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Throttling", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (WSS)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage (WSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Pod - Read & Writes)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Containers)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml new file mode 100644 index 0000000000..d77170afd8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-cluster' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-cluster.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - avg(rate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode=\"idle\"}[1m]))","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:windows_node_memory_MemFreeCached_bytes:sum{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Private Working Set)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Namespace","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Cluster(Windows)","uid":"4d08557fd9391b100730f2494bccac68","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml new file mode 100644 index 0000000000..13a1fc3abd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-namespace' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-namespace.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Namespace(Windows)","uid":"490b402361724ab1d4c45666c1fa9b6f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml new file mode 100644 index 0000000000..6686e54053 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-pod' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-pod.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":6,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum by (container) (rate(windows_container_network_received_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Received : {{ container }}","refId":"A"},{"expr":"sort_desc(sum by (container) (rate(windows_container_network_transmitted_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Transmitted : {{ container }}","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network I/O","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Pod","multi":false,"name":"pod","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\",namespace=\"$namespace\"}, pod)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Pod(Windows)","uid":"40597a704a610e936dc6ed374a7ce023","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml new file mode 100644 index 0000000000..e2a63ae208 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -0,0 +1,2024 @@ +{{- /* +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Pod", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}, workload)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml new file mode 100644 index 0000000000..95d758ea2d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -0,0 +1,2189 @@ +{{- /* +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Workload", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..d9ce9d738c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-cluster-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-cluster-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\"} * node:windows_node_num_cpu:sum{cluster=\"$cluster\"} / scalar(sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"}))","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:ratio{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O Pages)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(node:windows_node:sum{cluster=\"$cluster\"})","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (instance)(node:windows_node_filesystem_usage:{cluster=\"$cluster\"})\n","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Capacity","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Cluster(Windows)","uid":"53a43377ec9aaf2ff64dfc7a1f539334","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml new file mode 100644 index 0000000000..a7608496a3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-node-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-node-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (core) (irate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode!=\"idle\", instance=\"$instance\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{core}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage Per Core","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Memory","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation %","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"max(\n windows_os_visible_memory_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n - windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"memory used","refId":"A"},{"expr":"max(node:windows_node_memory_totalCached_bytes:sum{cluster=\"$cluster\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory cached","refId":"B"},{"expr":"max(windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory free","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Swap IO","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O) Pages","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"read","yaxis":1},{"alias":"io time","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"max(rate(windows_logical_disk_read_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"read","refId":"A"},{"expr":"max(rate(windows_logical_disk_write_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"written","refId":"B"},{"expr":"max(rate(windows_logical_disk_read_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]) + rate(windows_logical_disk_write_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ms","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Saturation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Net","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}\n","format":"time_series","legendFormat":"{{volume}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(windows_system_system_up_time{cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Node(Windows)","uid":"96e7484b0bb53b74fbc2bcb7723cd40b","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml new file mode 100644 index 0000000000..74a5303f8f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml @@ -0,0 +1,2256 @@ +{{- /* +Generated from 'kubelet' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubelet.enabled" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "kubelet" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + kubelet.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_node_name{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Running Kubelets", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Pods", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Containers", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 5, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Actual Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Desired Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 7, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Config Error Count", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 77 + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 77 + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 77 + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Kubelet", + "uid": "3138fa155d5915769fbded898ac09fd9", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml new file mode 100644 index 0000000000..f5c72844fb --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -0,0 +1,1464 @@ +{{- /* +Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-pod.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "pod", + "value": "pod" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Pods)", + "uid": "8b7a8b326d7a6f1f04244066368c67af", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml new file mode 100644 index 0000000000..801b09c265 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -0,0 +1,1736 @@ +{{- /* +Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-workload.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "workload", + "value": "workload" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=$__cell", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Workload)", + "uid": "bbb2a765a623ae38130206c7d94a160f", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..9869a3d3e0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -0,0 +1,1063 @@ +{{- /* +Generated from 'node-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-cluster-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "((\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} instance {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"})))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Cluster", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml new file mode 100644 index 0000000000..75e69afa22 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -0,0 +1,1089 @@ +{{- /* +Generated from 'node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major page Faults", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_exporter_build_info{job=\"node-exporter\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Node", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml new file mode 100644 index 0000000000..fe11875324 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml @@ -0,0 +1,1073 @@ +{{- /* +Generated from 'nodes-darwin' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.darwin.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes-darwin" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes-darwin.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Physical Memory", + "refId": "A" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory Used", + "refId": "B" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "App Memory", + "refId": "C" + }, + { + "expr": "node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Wired Memory", + "refId": "D" + }, + { + "expr": "node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compressed", + "refId": "E" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "(\n (\n avg(node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"}) -\n avg(node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"})\n ) /\n avg(node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"})\n)\n*\n100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / MacOS", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml new file mode 100644 index 0000000000..0da40a7b99 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml @@ -0,0 +1,1066 @@ +{{- /* +Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.linux.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"})\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname!=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / Nodes", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml new file mode 100644 index 0000000000..4d1e33208b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -0,0 +1,587 @@ +{{- /* +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml new file mode 100644 index 0000000000..9a7e7d0603 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml @@ -0,0 +1,1228 @@ +{{- /* +Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "pod-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + pod-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 8, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Pod", + "uid": "7a18067ce943a40ae25454675c19ff5c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml new file mode 100644 index 0000000000..5c11900e69 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -0,0 +1,1674 @@ +{{- /* +Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus-remote-write" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus-remote-write.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} != 0)\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n, 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Min Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Desired Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shards", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shard Details", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}consumer{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Remote Write Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Segments", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Retried Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Enqueue Retries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Misc. Rates", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": true, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "url", + "options": [ + + ], + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Remote Write", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml new file mode 100644 index 0000000000..27f7c44e2c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml @@ -0,0 +1,1235 @@ +{{- /* +Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "s" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}scrape_job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}interval{{`}}`}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded body size limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}slice{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=\"prometheus-k8s\",namespace=\"monitoring\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Overview", + "uid": "", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml new file mode 100644 index 0000000000..410812451e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml @@ -0,0 +1,1276 @@ +{{- /* +Generated from 'proxy' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "proxy" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + proxy.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rules Sync Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rule Sync Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Proxy", + "uid": "632e265de029684c40b21cb76bca4f94", + "version": 0 + } +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml new file mode 100644 index 0000000000..ee0cf08b2f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml @@ -0,0 +1,1118 @@ +{{- /* +Generated from 'scheduler' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "scheduler" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + scheduler.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml new file mode 100644 index 0000000000..5aafccdebe --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml @@ -0,0 +1,1438 @@ +{{- /* +Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "workload-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + workload-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 14, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Workload", + "uid": "728bf77cc1166d2f3133bf25846876cc", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/namespaces.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/namespaces.yaml new file mode 100644 index 0000000000..39ed210ed4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/namespaces.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + name: {{ .Values.grafana.defaultDashboards.namespace }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + annotations: +{{- if not .Values.grafana.defaultDashboards.cleanupOnUninstall }} + helm.sh/resource-policy: "keep" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl new file mode 100644 index 0000000000..6ae9dc72e6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl @@ -0,0 +1,7 @@ +{{/* Generate basic labels for prometheus-operator */}} +{{- define "kube-prometheus-stack.prometheus-operator.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app: {{ template "kube-prometheus-stack.name" . }}-operator +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl new file mode 100644 index 0000000000..f419caf54b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl @@ -0,0 +1,6 @@ +{{/* Generate basic labels for prometheus-operator-webhook */}} +{{- define "kube-prometheus-stack.prometheus-operator-webhook.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator-webhook +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml new file mode 100644 index 0000000000..054eac4a77 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml @@ -0,0 +1,143 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.annotations | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.prometheusOperator.admissionWebhooks.deployment.replicas }} + revisionHistoryLimit: {{ .Values.prometheusOperator.admissionWebhooks.deployment.revisionHistoryLimit }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podLabels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: prometheus-operator-admission-webhook + {{- $operatorRegistry := .Values.global.imageRegistry | default .Values.prometheusOperator.admissionWebhooks.deployment.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + - --log-format={{ .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + - --log-level={{ .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - "--web.enable-tls=true" + - "--web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }}" + - "--web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }}" + - "--web.listen-address=:{{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }}" + - "--web.tls-min-version={{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.tlsMinVersion }}" + ports: + - containerPort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.failureThreshold }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.containerSecurityContext | indent 12 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + volumeMounts: + - name: tls-secret + mountPath: /cert + readOnly: true + volumes: + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }}-webhook + automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml new file mode 100644 index 0000000000..52dd78f624 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget -}} +apiVersion: policy/v1{{ ternary "" "beta1" ($.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml new file mode 100644 index 0000000000..b06c129123 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.admissionWebhooks.deployment.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml new file mode 100644 index 0000000000..55511da36b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ template "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | indent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml new file mode 100644 index 0000000000..f7543b0f1a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml new file mode 100644 index 0000000000..4e3b0d9225 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml new file mode 100644 index 0000000000..b81257c168 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") (or .Values.global.cattle.psp.enabled .Values.global.rbac.pspEnabled) }} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} + - apiGroups: ['policy'] +{{- else }} + - apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml new file mode 100644 index 0000000000..4cf1335b22 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml new file mode 100644 index 0000000000..baed83db48 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -0,0 +1,73 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: create + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | replace "\n" "," }} + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + {{- with .Values.prometheusOperator.admissionWebhooks.createSecretJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml new file mode 100644 index 0000000000..5639cc9e80 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -0,0 +1,74 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: patch + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - patch + - --webhook-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --patch-failure-policy={{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- with .Values.prometheusOperator.admissionWebhooks.patchWebhookJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml new file mode 100644 index 0000000000..864deb52a0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml new file mode 100644 index 0000000000..076c467004 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml new file mode 100644 index 0000000000..0113b6a5d8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,47 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- if .Values.global.rbac.pspAnnotations }} +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml new file mode 100644 index 0000000000..f15abf4395 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml new file mode 100644 index 0000000000..30bde920b6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml new file mode 100644 index 0000000000..02594547d1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..da01f3b57e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/mutate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..4827871cca --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/validate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/certmanager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/certmanager.yaml new file mode 100644 index 0000000000..cb27e49f48 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/certmanager.yaml @@ -0,0 +1,55 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled -}} +{{- if not .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef -}} +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }} + issuerRef: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.kube-prometheus-stack" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + ca: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert +{{- end }} +--- +# generate a server certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }} + issuerRef: + {{- if .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef }} + {{- toYaml .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef | nindent 4 }} + {{- else }} + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + {{- end }} + dnsNames: + {{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | splitList "\n" | toYaml | nindent 4 }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..07e2e99967 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml @@ -0,0 +1,40 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort | quote }} + {{- else }} + - port: "8080" + {{- end }} + protocol: "TCP" + {{- if not .Values.prometheusOperator.tls.enabled }} + rules: + http: + - method: "GET" + path: "/metrics" + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrole.yaml new file mode 100644 index 0000000000..fd11b69eed --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrole.yaml @@ -0,0 +1,109 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - alertmanagers/finalizers + - alertmanagers/status + - alertmanagerconfigs + - prometheuses + - prometheuses/finalizers + - prometheuses/status + - prometheusagents + - prometheusagents/finalizers + - prometheusagents/status + - thanosrulers + - thanosrulers/finalizers + - thanosrulers/status + - scrapeconfigs + - servicemonitors + - podmonitors + - probes + - prometheusrules + verbs: + - '*' +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - '*' +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete +- apiGroups: + - "" + resources: + - services + - services/finalizers + - endpoints + verbs: + - get + - create + - update + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - patch + - create +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get +{{- if .Capabilities.APIVersions.Has "discovery.k8s.io/v1/EndpointSlice" }} +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml new file mode 100644 index 0000000000..ad9e3ef6c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/deployment.yaml new file mode 100644 index 0000000000..8a01b2912a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/deployment.yaml @@ -0,0 +1,204 @@ +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- $defaultKubeletSvcName := printf "%s-kubelet" (include "kube-prometheus-stack.fullname" .) }} +{{- if .Values.prometheusOperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.labels }} +{{ toYaml .Values.prometheusOperator.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.annotations | indent 4 }} +{{- end }} +spec: + replicas: 1 + revisionHistoryLimit: {{ .Values.prometheusOperator.revisionHistoryLimit }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- with .Values.prometheusOperator.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.podLabels }} +{{ toYaml .Values.prometheusOperator.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: {{ template "kube-prometheus-stack.name" . }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- $configReloaderRegistry := $base_registry | default .Values.prometheusOperator.prometheusConfigReloader.image.registry -}} + {{- $operatorRegistry := $base_registry | default .Values.prometheusOperator.image.registry -}} + {{- $thanosRegistry := $base_registry | default .Values.prometheusOperator.thanosImage.registry -}} + {{- if .Values.prometheusOperator.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.kubeletService.enabled }} + - --kubelet-service={{ .Values.prometheusOperator.kubeletService.namespace }}/{{ default $defaultKubeletSvcName .Values.prometheusOperator.kubeletService.name }} + {{- end }} + {{- if .Values.prometheusOperator.logFormat }} + - --log-format={{ .Values.prometheusOperator.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.logLevel }} + - --log-level={{ .Values.prometheusOperator.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - --deny-namespaces={{ tpl (.Values.prometheusOperator.denyNamespaces | join ",") $ }} + {{- end }} + {{- with $.Values.prometheusOperator.namespaces }} + {{- $namespaces := list }} + {{- if .releaseNamespace }} + {{- $namespaces = append $namespaces $namespace }} + {{- end }} + {{- if .additional }} + {{- range $ns := .additional }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + - --localhost=127.0.0.1 + {{- if .Values.prometheusOperator.prometheusDefaultBaseImage }} + - --prometheus-default-base-image={{ $base_registry | default .Values.prometheusOperator.prometheusDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.prometheusDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + - --alertmanager-default-base-image={{ $base_registry | default .Values.prometheusOperator.alertmanagerDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + {{- else }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }} + {{- end }} + - --config-reloader-cpu-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).cpu) | default 0 }} + - --config-reloader-cpu-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).cpu) | default 0 }} + - --config-reloader-memory-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).memory) | default 0 }} + - --config-reloader-memory-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).memory) | default 0 }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.enableProbe }} + - --enable-config-reloader-probes=true + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceNamespaces }} + - --alertmanager-instance-namespaces={{ .Values.prometheusOperator.alertmanagerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceSelector }} + - --alertmanager-instance-selector={{ .Values.prometheusOperator.alertmanagerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerConfigNamespaces }} + - --alertmanager-config-namespaces={{ .Values.prometheusOperator.alertmanagerConfigNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceNamespaces }} + - --prometheus-instance-namespaces={{ .Values.prometheusOperator.prometheusInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceSelector }} + - --prometheus-instance-selector={{ .Values.prometheusOperator.prometheusInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.thanosImage.sha }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}@sha256:{{ .Values.prometheusOperator.thanosImage.sha }} + {{- else }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceNamespaces }} + - --thanos-ruler-instance-namespaces={{ .Values.prometheusOperator.thanosRulerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceSelector }} + - --thanos-ruler-instance-selector={{ .Values.prometheusOperator.thanosRulerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.secretFieldSelector }} + - --secret-field-selector={{ tpl (.Values.prometheusOperator.secretFieldSelector) $ }} + {{- end }} + {{- if .Values.prometheusOperator.clusterDomain }} + - --cluster-domain={{ .Values.prometheusOperator.clusterDomain }} + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - --web.enable-tls=true + - --web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }} + - --web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }} + - --web.listen-address=:{{ .Values.prometheusOperator.tls.internalPort }} + - --web.tls-min-version={{ .Values.prometheusOperator.tls.tlsMinVersion }} + ports: + - containerPort: {{ .Values.prometheusOperator.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + env: + {{- range $key, $value := .Values.prometheusOperator.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.containerSecurityContext | indent 12 }} + volumeMounts: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + mountPath: /cert + readOnly: true + {{- end }} + {{- with .Values.prometheusOperator.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + {{- end }} + {{- with .Values.prometheusOperator.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.prometheusOperator.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.tolerations }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/networkpolicy.yaml new file mode 100644 index 0000000000..cfd5b0b8c7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/networkpolicy.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + egress: + - {} + ingress: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort }} + {{- else }} + - port: 8080 + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml new file mode 100644 index 0000000000..61bc3d9040 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.operator.fullname" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..40e0fc5c15 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp.yaml new file mode 100644 index 0000000000..28a9075d3e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: {{ .Values.prometheusOperator.hostNetwork }} + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/service.yaml new file mode 100644 index 0000000000..d45ab22d08 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/service.yaml @@ -0,0 +1,57 @@ +{{- if .Values.prometheusOperator.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.service.labels }} +{{ toYaml .Values.prometheusOperator.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/serviceaccount.yaml new file mode 100644 index 0000000000..4f84974f9b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +automountServiceAccountToken: {{ .Values.prometheusOperator.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/servicemonitor.yaml new file mode 100644 index 0000000000..cbe79e1253 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- with .Values.prometheusOperator.serviceMonitor.additionalLabels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheusOperator.serviceMonitor | nindent 2 }} + endpoints: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: https + scheme: https + tlsConfig: + serverName: {{ template "kube-prometheus-stack.operator.fullname" . }} + ca: + secret: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + key: {{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}ca.crt{{ else }}ca{{ end }} + optional: false + {{- else }} + - port: http + {{- end }} + honorLabels: true + {{- if .Values.prometheusOperator.serviceMonitor.interval }} + interval: {{ .Values.prometheusOperator.serviceMonitor.interval }} + {{- end }} + metricRelabelings: + {{- if .Values.prometheusOperator.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.prometheusOperator.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheusOperator.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheusOperator.serviceMonitor.relabelings | indent 6 }} +{{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..f225d16dde --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.prometheusOperator.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + {{- with .Values.prometheusOperator.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-prometheus-stack.name" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/_rules.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/_rules.tpl new file mode 100644 index 0000000000..4a8213d089 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/_rules.tpl @@ -0,0 +1,44 @@ +{{- /* +Generated file. Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- define "rules.names" }} +rules: + - "alertmanager.rules" + - "config-reloaders" + - "etcd" + - "general.rules" + - "k8s.rules.container-cpu-usage-seconds-total" + - "k8s.rules.container-memory-cache" + - "k8s.rules.container-memory-rss" + - "k8s.rules.container-memory-swap" + - "k8s.rules.container-memory-working-set-bytes" + - "k8s.rules.container-resource" + - "k8s.rules.pod-owner" + - "kube-apiserver-availability.rules" + - "kube-apiserver-burnrate.rules" + - "kube-apiserver-histogram.rules" + - "kube-apiserver-slos" + - "kube-prometheus-general.rules" + - "kube-prometheus-node-recording.rules" + - "kube-scheduler.rules" + - "kube-state-metrics" + - "kubelet.rules" + - "kubernetes-apps" + - "kubernetes-resources" + - "kubernetes-storage" + - "kubernetes-system" + - "kubernetes-system-kube-proxy" + - "kubernetes-system-apiserver" + - "kubernetes-system-kubelet" + - "kubernetes-system-controller-manager" + - "kubernetes-system-scheduler" + - "node-exporter.rules" + - "node-exporter" + - "node.rules" + - "node-network" + - "prometheus-operator" + - "prometheus" + - "windows.node.rules" + - "windows.pod.rules" +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml new file mode 100644 index 0000000000..bff930981a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-relabel-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alert-relabel-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml new file mode 100644 index 0000000000..2fe8fdb816 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alertmanager-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs) . | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml new file mode 100644 index 0000000000..cb4aabaa7b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml @@ -0,0 +1,43 @@ +{{- if or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-additional-prometheus-rules + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- if .Values.additionalPrometheusRulesMap }} +{{- range $prometheusRuleName, $prometheusRule := .Values.additionalPrometheusRulesMap }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ $prometheusRuleName }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $prometheusRule.additionalLabels }} +{{ toYaml $prometheusRule.additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml $prometheusRule.groups| indent 8 }} +{{- end }} +{{- else }} +{{- range .Values.additionalPrometheusRules }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml .groups| indent 8 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml new file mode 100644 index 0000000000..ebdf766fde --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-scrape-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if eq ( typeOf .Values.prometheus.prometheusSpec.additionalScrapeConfigs ) "string" }} + additional-scrape-configs.yaml: {{ tpl .Values.prometheus.prometheusSpec.additionalScrapeConfigs $ | b64enc | quote }} +{{- else }} + additional-scrape-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalScrapeConfigs) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..74d61d7c13 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} +spec: + endpointSelector: + {{- if .Values.prometheus.networkPolicy.cilium.endpointSelector }} + {{- toYaml .Values.prometheus.networkPolicy.cilium.endpointSelector | nindent 4 }} + {{- else }} + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.egress }} + egress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.egress | nindent 4 }} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.ingress }} + ingress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.ingress | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrole.yaml new file mode 100644 index 0000000000..3585b5db11 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrole.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +# This permission are not in the kube-prometheus repo +# they're grabbed from https://github.com/prometheus/prometheus/blob/master/documentation/examples/rbac-setup.yml +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - "networking.k8s.io" + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +{{- if .Values.prometheus.additionalRulesForClusterRole }} +{{ toYaml .Values.prometheus.additionalRulesForClusterRole | indent 0 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrolebinding.yaml new file mode 100644 index 0000000000..9fc4f65da4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/csi-secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/csi-secret.yaml new file mode 100644 index 0000000000..e05382f633 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/csi-secret.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.secretProviderClass }} +--- +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +spec: +{{ toYaml .Values.prometheus.prometheusSpec.thanos.secretProviderClass | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/extrasecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/extrasecret.yaml new file mode 100644 index 0000000000..17f3478a46 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.prometheus.extraSecret.data -}} +{{- $secretName := printf "prometheus-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.prometheus.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.extraSecret.annotations }} + annotations: +{{ toYaml .Values.prometheus.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.prometheus.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingress.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingress.yaml new file mode 100644 index 0000000000..d2f6af5dd1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}} + {{- $pathType := .Values.prometheus.ingress.pathType | default "ImplementationSpecific" -}} + {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" -}} + {{- $servicePort := .Values.prometheus.ingress.servicePort | default .Values.prometheus.service.port -}} + {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} + {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} + {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} + {{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.ingress.annotations) . | nindent 4 }} +{{- end }} + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.ingress.labels }} +{{ toYaml .Values.prometheus.ingress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.ingress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.ingress.hosts }} + {{- range $host := .Values.prometheus.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.ingress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml new file mode 100644 index 0000000000..3f507cfa9f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosIngress.enabled }} +{{- $pathType := .Values.prometheus.thanosIngress.pathType | default "" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "thanos-discovery" }} +{{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} +{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} +{{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.thanosIngress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.thanosIngress.annotations) . | nindent 4 }} +{{- end }} + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-gateway + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosIngress.labels }} +{{ toYaml .Values.prometheus.thanosIngress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.thanosIngress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.thanosIngress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.thanosIngress.hosts }} + {{- range $host := .Values.prometheus.thanosIngress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.thanosIngress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.thanosIngress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressperreplica.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressperreplica.yaml new file mode 100644 index 0000000000..1d76d135c8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled .Values.prometheus.ingressPerReplica.enabled }} +{{- $pathType := .Values.prometheus.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $servicePort := .Values.prometheus.servicePerReplica.port -}} +{{- $ingressValues := .Values.prometheus.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" $ }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/networkpolicy.yaml new file mode 100644 index 0000000000..1296a79063 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/networkpolicy.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + {{- if .Values.prometheus.networkPolicy.egress }} + egress: + {{- toYaml .Values.prometheus.networkPolicy.egress | nindent 4 }} + {{- end }} + {{- if .Values.prometheus.networkPolicy.ingress }} + ingress: + {{- toYaml .Values.prometheus.networkPolicy.ingress | nindent 4 }} + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + {{- if .Values.prometheus.networkPolicy.podSelector }} + {{- toYaml .Values.prometheus.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/nginx-config.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/nginx-config.yaml new file mode 100644 index 0000000000..e4d91f9a9e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/nginx-config.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-nginx-proxy-config + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + server { + listen 8081; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:9090/; + + sub_filter_once off; + sub_filter 'var PATH_PREFIX = "";' 'var PATH_PREFIX = ".";'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podDisruptionBudget.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podDisruptionBudget.yaml new file mode 100644 index 0000000000..48f3f1f5a6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podDisruptionBudget.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.prometheus.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.prometheus.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.prometheus.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.prometheus.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podmonitors.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podmonitors.yaml new file mode 100644 index 0000000000..4e748c23b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podmonitors.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalPodMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalPodMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + podMetricsEndpoints: +{{ toYaml .podMetricsEndpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .sampleLimit }} + sampleLimit: {{ .sampleLimit }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/prometheus.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/prometheus.yaml new file mode 100644 index 0000000000..5c3c8d4d1f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/prometheus.yaml @@ -0,0 +1,472 @@ +{{- if .Values.prometheus.enabled }} +{{- if .Values.prometheus.agentMode }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: PrometheusAgent +{{- else }} +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +{{- end }} +metadata: + name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +spec: +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.alertingEndpoints .Values.alertmanager.enabled) }} + alerting: + alertmanagers: +{{- if .Values.prometheus.prometheusSpec.alertingEndpoints }} +{{ toYaml .Values.prometheus.prometheusSpec.alertingEndpoints | indent 6 }} +{{- else if .Values.alertmanager.enabled }} + - namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + pathPrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.scheme }} + scheme: {{ .Values.alertmanager.alertmanagerSpec.scheme }} + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.tlsConfig }} + tlsConfig: +{{ toYaml .Values.alertmanager.alertmanagerSpec.tlsConfig | indent 10 }} + {{- end }} + apiVersion: {{ .Values.alertmanager.apiVersion }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.apiserverConfig }} + apiserverConfig: +{{ toYaml .Values.prometheus.prometheusSpec.apiserverConfig | indent 4}} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.prometheus.prometheusSpec.image.registry -}} + {{- if and .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}" + {{- end }} + version: {{ default .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.version }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.prometheus.prometheusSpec.additionalArgs | indent 4}} +{{- end -}} +{{- if .Values.prometheus.prometheusSpec.externalLabels }} + externalLabels: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusExternalLabelNameClear }} + prometheusExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.prometheusExternalLabelName }} + prometheusExternalLabelName: "{{ .Values.prometheus.prometheusSpec.prometheusExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.replicaExternalLabelNameClear }} + replicaExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.replicaExternalLabelName }} + replicaExternalLabelName: "{{ .Values.prometheus.prometheusSpec.replicaExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} + enableRemoteWriteReceiver: {{ .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalUrl }} + externalUrl: "{{ tpl .Values.prometheus.prometheusSpec.externalUrl . }}" +{{- else if and .Values.prometheus.ingress.enabled .Values.prometheus.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.prometheus.ingress.hosts 0) . }}{{ .Values.prometheus.prometheusSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "kube-prometheus-stack.namespace" . }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.nodeSelector }} +{{ toYaml .Values.prometheus.prometheusSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.prometheus.prometheusSpec.paused }} + replicas: {{ .Values.prometheus.prometheusSpec.replicas }} + shards: {{ .Values.prometheus.prometheusSpec.shards }} + logLevel: {{ .Values.prometheus.prometheusSpec.logLevel }} + logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} + listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} +{{- if not .Values.prometheus.agentMode }} + enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.web }} + web: +{{ toYaml .Values.prometheus.prometheusSpec.web | indent 4 }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.exemplars }} + exemplars: + {{ toYaml .Values.prometheus.prometheusSpec.exemplars | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableFeatures }} + enableFeatures: +{{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }} + - {{ tpl $enableFeatures $ }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeInterval }} + scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.prometheusSpec.scrapeTimeout }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.evaluationInterval }} + evaluationInterval: {{ .Values.prometheus.prometheusSpec.evaluationInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.resources }} + resources: +{{ toYaml .Values.prometheus.prometheusSpec.resources | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} + retention: {{ .Values.prometheus.prometheusSpec.retention | quote }} +{{- if .Values.prometheus.prometheusSpec.retentionSize }} + retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tsdb }} + tsdb: + {{- if .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + outOfOrderTimeWindow: {{ .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + {{- end }} +{{- end }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.walCompression false }} + walCompression: false +{{ else }} + walCompression: true +{{- end }} +{{- if .Values.prometheus.prometheusSpec.routePrefix }} + routePrefix: {{ .Values.prometheus.prometheusSpec.routePrefix | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.secrets }} + secrets: +{{ toYaml .Values.prometheus.prometheusSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.configMaps }} + configMaps: +{{ toYaml .Values.prometheus.prometheusSpec.configMaps | indent 4 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorSelector }} + serviceMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues }} + serviceMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + serviceMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector }} + serviceMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector | indent 4) . }} +{{ else }} + serviceMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorSelector }} + podMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues }} + podMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + podMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector }} + podMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector | indent 4) . }} +{{ else }} + podMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeSelector }} + probeSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.probeSelectorNilUsesHelmValues }} + probeSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + probeSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeNamespaceSelector }} + probeNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeNamespaceSelector | indent 4) . }} +{{ else }} + probeNamespaceSelector: {} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.remoteRead .Values.prometheus.prometheusSpec.additionalRemoteRead) }} + remoteRead: +{{- if .Values.prometheus.prometheusSpec.remoteRead }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteRead | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteRead }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteRead | indent 4 }} +{{- end }} +{{- end }} +{{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite) }} + remoteWrite: +{{- if .Values.prometheus.prometheusSpec.remoteWrite }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteWrite }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteWrite | indent 4 }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.prometheusSpec.securityContext | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleSelector | indent 4) . }} +{{- else if .Values.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigSelector }} + scrapeConfigSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.scrapeConfigSelectorNilUsesHelmValues }} + scrapeConfigSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + scrapeConfigSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector }} + scrapeConfigNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector | indent 4) . }} +{{ else }} + scrapeConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.storageSpec }} + storage: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.storageSpec | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMetadata }} + podMetadata: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMetadata | indent 4) . }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.query }} + query: +{{ toYaml .Values.prometheus.prometheusSpec.query | indent 4}} +{{- end }} +{{- if or .Values.prometheus.prometheusSpec.podAntiAffinity .Values.prometheus.prometheusSpec.affinity }} + affinity: +{{- if .Values.prometheus.prometheusSpec.affinity }} +{{ toYaml .Values.prometheus.prometheusSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.tolerations }} +{{ toYaml .Values.prometheus.prometheusSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.prometheus.prometheusSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} + additionalScrapeConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + key: additional-scrape-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.enabled }} + additionalScrapeConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.key }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if or .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + additionalAlertManagerConfigs: +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + key: additional-alertmanager-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + name: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.key }} + {{- if hasKey .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret "optional" }} + optional: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.optional }} + {{- end }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} + additionalAlertRelabelConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + key: additional-alert-relabel-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret }} + additionalAlertRelabelConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.key }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.containers }} + containers: +{{ tpl .Values.prometheus.prometheusSpec.containers $ | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.initContainers }} + initContainers: +{{ toYaml .Values.prometheus.prometheusSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.priorityClassName }} + priorityClassName: {{ .Values.prometheus.prometheusSpec.priorityClassName }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.thanos }} + thanos: +{{- with (omit .Values.prometheus.prometheusSpec.thanos "objectStorageConfig")}} +{{ toYaml . | indent 4 }} +{{- end }} +{{- if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).existingSecret) }} + objectStorageConfig: + key: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.name }}" +{{- else if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).secret) }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.disableCompaction }} + disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }} +{{- end }} +{{- end }} + portName: {{ .Values.prometheus.prometheusSpec.portName }} +{{- if .Values.prometheus.prometheusSpec.volumes }} + volumes: +{{ toYaml .Values.prometheus.prometheusSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.prometheus.prometheusSpec.volumeMounts | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs }} + arbitraryFSAccessThroughSMs: +{{ toYaml .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorLabels }} + overrideHonorLabels: {{ .Values.prometheus.prometheusSpec.overrideHonorLabels }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} + overrideHonorTimestamps: {{ .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + ignoreNamespaceSelectors: {{ .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} + enforcedNamespaceLabel: {{ .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} +{{- $prometheusDefaultRulesExcludedFromEnforce := (include "rules.names" .) | fromYaml }} +{{- if not .Values.prometheus.agentMode }} + prometheusRulesExcludedFromEnforce: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - ruleNamespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + ruleName: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce }} +{{ toYaml .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce | indent 4 }} +{{- end }} +{{- end }} + excludedFromEnforcement: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - group: monitoring.coreos.com + resource: prometheusrules + namespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + name: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.excludedFromEnforcement }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.excludedFromEnforcement | indent 4) . }} +{{- end }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.queryLogFile }} + queryLogFile: {{ .Values.prometheus.prometheusSpec.queryLogFile }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.sampleLimit }} + sampleLimit: {{ .Values.prometheus.prometheusSpec.sampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} + enforcedKeepDroppedTargets: {{ .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedSampleLimit }} + enforcedSampleLimit: {{ .Values.prometheus.prometheusSpec.enforcedSampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedTargetLimit }} + enforcedTargetLimit: {{ .Values.prometheus.prometheusSpec.enforcedTargetLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelLimit }} + enforcedLabelLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} + enforcedLabelNameLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit}} + enforcedLabelValueLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} + allowOverlappingBlocks: {{ .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.minReadySeconds }} + minReadySeconds: {{ .Values.prometheus.prometheusSpec.minReadySeconds }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} + maximumStartupDurationSeconds: {{ .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} +{{- end }} + hostNetwork: {{ .Values.prometheus.prometheusSpec.hostNetwork }} +{{- if .Values.prometheus.prometheusSpec.hostAliases }} + hostAliases: +{{ toYaml .Values.prometheus.prometheusSpec.hostAliases | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tracingConfig }} + tracingConfig: +{{ toYaml .Values.prometheus.prometheusSpec.tracingConfig | indent 4 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrole.yaml new file mode 100644 index 0000000000..71476cd18b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrole.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..a393928c78 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp.yaml new file mode 100644 index 0000000000..62d3854151 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' +{{- if .Values.prometheus.podSecurityPolicy.volumes }} +{{ toYaml .Values.prometheus.podSecurityPolicy.volumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- if .Values.prometheus.podSecurityPolicy.allowedCapabilities }} + allowedCapabilities: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedCapabilities | indent 4 }} +{{- end }} +{{- if .Values.prometheus.podSecurityPolicy.allowedHostPaths }} + allowedHostPaths: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedHostPaths | indent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml new file mode 100644 index 0000000000..b66f052ade --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -0,0 +1,305 @@ +{{- /* +Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/alertmanager-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} +{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: alertmanager.rules + rules: +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedReload | default false) }} + - alert: AlertmanagerFailedReload + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Configuration has failed to load for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedreload + summary: Reloading an Alertmanager configuration has failed. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "AlertmanagerFailedReload" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedReload" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerMembersInconsistent | default false) }} + - alert: AlertmanagerMembersInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} has only found {{`{{`}} $value {{`}}`}} members of the {{`{{`}}$labels.job{{`}}`}} cluster. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagermembersinconsistent + summary: A member of an Alertmanager cluster has not found all other cluster members. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + < on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) group_left + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) (max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m])) + for: {{ dig "AlertmanagerMembersInconsistent" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerMembersInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedToSendAlerts | default false) }} + - alert: AlertmanagerFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} failed to send {{`{{`}} $value | humanizePercentage {{`}}`}} of notifications to {{`{{`}} $labels.integration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedtosendalerts + summary: An Alertmanager instance failed to send notifications. + expr: |- + ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerConfigInconsistent | default false) }} + - alert: AlertmanagerConfigInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have different configurations. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerconfiginconsistent + summary: Alertmanager instances within the same cluster have different configurations. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + count_values by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + ) + != 1 + for: {{ dig "AlertmanagerConfigInconsistent" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerConfigInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterDown | default false) }} + - alert: AlertmanagerClusterDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have been up for less than half of the last 5m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterdown + summary: Half or more of the Alertmanager instances within the same cluster are down. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + avg_over_time(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) < 0.5 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterCrashlooping | default false) }} + - alert: AlertmanagerClusterCrashlooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have restarted at least 5 times in the last 10m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclustercrashlooping + summary: Half or more of the Alertmanager instances within the same cluster are crashlooping. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + changes(process_start_time_seconds{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[10m]) > 4 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterCrashlooping" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterCrashlooping" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml new file mode 100644 index 0000000000..8416d6df40 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml @@ -0,0 +1,57 @@ +{{- /* +Generated from 'config-reloaders' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.configReloaders }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "config-reloaders" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: config-reloaders + rules: +{{- if not (.Values.defaultRules.disabled.ConfigReloaderSidecarErrors | default false) }} + - alert: ConfigReloaderSidecarErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders | indent 8 }} +{{- end }} + description: 'Errors encountered while the {{`{{`}}$labels.pod{{`}}`}} config-reloader sidecar attempts to sync config in {{`{{`}}$labels.namespace{{`}}`}} namespace. + + As a result, configuration for service running in {{`{{`}}$labels.pod{{`}}`}} may be stale and cannot be updated anymore.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/configreloadersidecarerrors + summary: config-reloader sidecar has not had a successful reload for 10m + expr: max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 + for: {{ dig "ConfigReloaderSidecarErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "ConfigReloaderSidecarErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml new file mode 100644 index 0000000000..a1d7a508f8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml @@ -0,0 +1,461 @@ +{{- /* +Generated from 'etcd' group from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.etcd }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: etcd + rules: +{{- if not (.Values.defaultRules.disabled.etcdMembersDown | default false) }} + - alert: etcdMembersDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster members are down. + expr: |- + max without (endpoint) ( + sum without (instance) (up{job=~".*etcd.*"} == bool 0) + or + count without (To) ( + sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + ) + ) + > 0 + for: {{ dig "etcdMembersDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMembersDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdInsufficientMembers | default false) }} + - alert: etcdInsufficientMembers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster has insufficient number of members. + expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2) + for: {{ dig "etcdInsufficientMembers" "for" "3m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdInsufficientMembers" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdNoLeader | default false) }} + - alert: etcdNoLeader + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.' + summary: etcd cluster has no leader. + expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 + for: {{ dig "etcdNoLeader" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdNoLeader" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfLeaderChanges | default false) }} + - alert: etcdHighNumberOfLeaderChanges + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' + summary: etcd cluster has high number of leader changes. + expr: increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4 + for: {{ dig "etcdHighNumberOfLeaderChanges" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfLeaderChanges" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 1 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 5 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdGRPCRequestsSlow | default false) }} + - alert: etcdGRPCRequestsSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.' + summary: etcd grpc requests are slow + expr: |- + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) + > 0.15 + for: {{ dig "etcdGRPCRequestsSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdGRPCRequestsSlow" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdMemberCommunicationSlow | default false) }} + - alert: etcdMemberCommunicationSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster member communication is slow. + expr: |- + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.15 + for: {{ dig "etcdMemberCommunicationSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMemberCommunicationSlow" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedProposals | default false) }} + - alert: etcdHighNumberOfFailedProposals + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of proposal failures. + expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + for: {{ dig "etcdHighNumberOfFailedProposals" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedProposals" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.5 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 1 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighCommitDurations | default false) }} + - alert: etcdHighCommitDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile commit durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.25 + for: {{ dig "etcdHighCommitDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighCommitDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseQuotaLowSpace | default false) }} + - alert: etcdDatabaseQuotaLowSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' + summary: etcd cluster database is running full. + expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95 + for: {{ dig "etcdDatabaseQuotaLowSpace" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseQuotaLowSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdExcessiveDatabaseGrowth | default false) }} + - alert: etcdExcessiveDatabaseGrowth + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.' + summary: etcd cluster database growing very fast. + expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"} + for: {{ dig "etcdExcessiveDatabaseGrowth" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdExcessiveDatabaseGrowth" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseHighFragmentationRatio | default false) }} + - alert: etcdDatabaseHighFragmentationRatio + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size in use on instance {{`{{`}} $labels.instance {{`}}`}} is {{`{{`}} $value | humanizePercentage {{`}}`}} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.' + runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation + summary: etcd database size in use is less than 50% of the actual allocated storage. + expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600 + for: {{ dig "etcdDatabaseHighFragmentationRatio" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseHighFragmentationRatio" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml new file mode 100644 index 0000000000..8aca0b85f5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml @@ -0,0 +1,125 @@ +{{- /* +Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: general.rules + rules: +{{- if not (.Values.defaultRules.disabled.TargetDown | default false) }} + - alert: TargetDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/targetdown + summary: One or more targets are unreachable. + expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10 + for: {{ dig "TargetDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "TargetDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.Watchdog | default false) }} + - alert: Watchdog + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + + This alert is always firing, therefore it should always be firing in Alertmanager + + and always fire against a receiver. There are integrations with various notification + + mechanisms that send a notification when this alert is not firing. For example the + + "DeadMansSnitch" integration in PagerDuty. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager is working properly. + expr: vector(1) + labels: + severity: {{ dig "Watchdog" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.InfoInhibitor | default false) }} + - alert: InfoInhibitor + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert that is used to inhibit info alerts. + + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + + other alerts. + + This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a + + severity of ''warning'' or ''critical'' starts firing on the same namespace. + + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: {{ dig "InfoInhibitor" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml new file mode 100644 index 0000000000..9de5f5bc9c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml @@ -0,0 +1,43 @@ +{{- /* +Generated from 'k8s.rules.container-cpu-usage-seconds-total' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerCpuUsageSecondsTotal }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-cpu-usage-seconds-total" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_cpu_usage_seconds_total + rules: + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml new file mode 100644 index 0000000000..323f41f9cb --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-cache' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryCache }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-cache" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_cache + rules: + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml new file mode 100644 index 0000000000..312d73c889 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-rss' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryRss }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-rss" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_rss + rules: + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml new file mode 100644 index 0000000000..136595e801 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-swap' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemorySwap }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-swap" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_swap + rules: + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml new file mode 100644 index 0000000000..d308b7473a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-working-set-bytes' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryWorkingSetBytes }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-working-set-bytes" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_working_set_bytes + rules: + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml new file mode 100644 index 0000000000..2d896e59e4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml @@ -0,0 +1,168 @@ +{{- /* +Generated from 'k8s.rules.container-resource' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerResource }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-resource" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_resource + rules: + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml new file mode 100644 index 0000000000..4915b25e73 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml @@ -0,0 +1,107 @@ +{{- /* +Generated from 'k8s.rules.pod-owner' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sPodOwner }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.pod-owner" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.pod_owner + rules: + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) group_left(owner_name) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="{{ $kubeStateMetricsJob }}"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml new file mode 100644 index 0000000000..c61bd222ab --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -0,0 +1,237 @@ +{{- /* +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8s }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules + rules: + - expr: |- + sum by (cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( + 1, max by (replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml new file mode 100644 index 0000000000..6194e9c614 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -0,0 +1,273 @@ +{{- /* +Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverAvailability }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-availability.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - interval: 3m + name: kube-apiserver-availability.rules + rules: + - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30 + record: code_verb:apiserver_request_total:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (increase(apiserver_request_sli_duration_seconds_count{job="apiserver"}[1h])) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h])) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + # write too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + ( + # read too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + ) + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d) + labels: + verb: all + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + # too slow + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + ( + # too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml new file mode 100644 index 0000000000..e6666a6f41 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml @@ -0,0 +1,440 @@ +{{- /* +Generated from 'kube-apiserver-burnrate.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverBurnrate }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-burnrate.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-burnrate.rules + rules: + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml new file mode 100644 index 0000000000..d145341952 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml @@ -0,0 +1,53 @@ +{{- /* +Generated from 'kube-apiserver-histogram.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverHistogram }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-histogram.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-histogram.rules + rules: + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml new file mode 100644 index 0000000000..30ef9a4293 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -0,0 +1,159 @@ +{{- /* +Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverSlos }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-slos" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-slos + rules: +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) + and + sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) + and + sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 6h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 30m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) + and + sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 2h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) + and + sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "3h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 3d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 6h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml new file mode 100644 index 0000000000..fcf35f389b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml @@ -0,0 +1,49 @@ +{{- /* +Generated from 'kube-prometheus-general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusGeneral }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-general.rules + rules: + - expr: count without(instance, pod, node) (up == 1) + record: count:up1 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: count without(instance, pod, node) (up == 0) + record: count:up0 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml new file mode 100644 index 0000000000..7a0d202324 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml @@ -0,0 +1,93 @@ +{{- /* +Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-recording.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-node-recording.rules + rules: + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance) + record: instance:node_cpu:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) + record: instance:node_network_receive_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) + record: instance:node_network_transmit_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) + record: instance:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) + record: cluster:node_cpu:sum_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) + record: cluster:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml new file mode 100644 index 0000000000..c9d61ce37b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml @@ -0,0 +1,135 @@ +{{- /* +Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-scheduler.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-scheduler.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml new file mode 100644 index 0000000000..d1ad3cae5e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml @@ -0,0 +1,152 @@ +{{- /* +Generated from 'kube-state-metrics' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubeStateMetrics-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeStateMetrics }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-state-metrics" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-state-metrics + rules: +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsListErrors | default false) }} + - alert: KubeStateMetricsListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricslisterrors + summary: kube-state-metrics is experiencing errors in list operations. + expr: |- + (sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsListErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsWatchErrors | default false) }} + - alert: KubeStateMetricsWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricswatcherrors + summary: kube-state-metrics is experiencing errors in watch operations. + expr: |- + (sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsWatchErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardingMismatch | default false) }} + - alert: KubeStateMetricsShardingMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardingmismatch + summary: kube-state-metrics sharding is misconfigured. + expr: stdvar (kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) != 0 + for: {{ dig "KubeStateMetricsShardingMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardingMismatch" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardsMissing | default false) }} + - alert: KubeStateMetricsShardsMissing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardsmissing + summary: kube-state-metrics shards are missing. + expr: |- + 2^max(kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - 1 + - + sum( 2 ^ max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="{{ $kubeStateMetricsJob }}"}) ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + != 0 + for: {{ dig "KubeStateMetricsShardsMissing" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardsMissing" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml new file mode 100644 index 0000000000..39fdddf3fe --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml @@ -0,0 +1,65 @@ +{{- /* +Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubelet }} +{{- if (include "exporter.kubelet.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubelet.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubelet.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml new file mode 100644 index 0000000000..2a861a522c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -0,0 +1,568 @@ +{{- /* +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesApps }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-apps + rules: +{{- if not (.Values.defaultRules.disabled.KubePodCrashLooping | default false) }} + - alert: KubePodCrashLooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: 'Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is in waiting state (reason: "CrashLoopBackOff").' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodcrashlooping + summary: Pod is crash looping. + expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) >= 1 + for: {{ dig "KubePodCrashLooping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodCrashLooping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePodNotReady | default false) }} + - alert: KubePodNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodnotready + summary: Pod has been in a non-ready state for more than 15 minutes. + expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown|Failed"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left(owner_kind) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + for: {{ dig "KubePodNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentGenerationMismatch | default false) }} + - alert: KubeDeploymentGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentgenerationmismatch + summary: Deployment generation mismatch due to possible roll-back + expr: |- + kube_deployment_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeDeploymentGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentReplicasMismatch | default false) }} + - alert: KubeDeploymentReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_deployment_spec_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_deployment_status_replicas_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeDeploymentReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentRolloutStuck | default false) }} + - alert: KubeDeploymentRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Rollout of deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} is not progressing for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentrolloutstuck + summary: Deployment rollout is not progressing. + expr: |- + kube_deployment_status_condition{condition="Progressing", status="false",job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != 0 + for: {{ dig "KubeDeploymentRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetReplicasMismatch | default false) }} + - alert: KubeStatefulSetReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetreplicasmismatch + summary: StatefulSet has not matched the expected number of replicas. + expr: |- + ( + kube_statefulset_status_replicas_ready{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetGenerationMismatch | default false) }} + - alert: KubeStatefulSetGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetgenerationmismatch + summary: StatefulSet generation mismatch due to possible roll-back + expr: |- + kube_statefulset_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeStatefulSetGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetUpdateNotRolledOut | default false) }} + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetupdatenotrolledout + summary: StatefulSet update has not been rolled out. + expr: |- + ( + max without (revision) ( + kube_statefulset_status_current_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetUpdateNotRolledOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetUpdateNotRolledOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetRolloutStuck | default false) }} + - alert: KubeDaemonSetRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} has not finished or progressed for at least 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetrolloutstuck + summary: DaemonSet rollout is stuck. + expr: |- + ( + ( + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + 0 + ) or ( + kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeDaemonSetRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeContainerWaiting | default false) }} + - alert: KubeContainerWaiting + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontainerwaiting + summary: Pod container waiting longer than 1 hour + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) > 0 + for: {{ dig "KubeContainerWaiting" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeContainerWaiting" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetNotScheduled | default false) }} + - alert: KubeDaemonSetNotScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetnotscheduled + summary: DaemonSet pods are not scheduled. + expr: |- + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + - + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetNotScheduled" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetNotScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetMisScheduled | default false) }} + - alert: KubeDaemonSetMisScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetmisscheduled + summary: DaemonSet pods are misscheduled. + expr: kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetMisScheduled" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetMisScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobNotCompleted | default false) }} + - alert: KubeJobNotCompleted + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than {{`{{`}} "43200" | humanizeDuration {{`}}`}} to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobnotcompleted + summary: Job did not complete in time + expr: |- + time() - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, job_name, cluster) (kube_job_status_start_time{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + and + kube_job_status_active{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0) > 43200 + labels: + severity: {{ dig "KubeJobNotCompleted" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobFailed | default false) }} + - alert: KubeJobFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. Removing failed job after investigation should clear this alert. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobfailed + summary: Job failed to complete. + expr: kube_job_failed{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeJobFailed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeJobFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaReplicasMismatch | default false) }} + - alert: KubeHpaReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpareplicasmismatch + summary: HPA has not matched desired number of replicas. + expr: |- + (kube_horizontalpodautoscaler_status_desired_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_horizontalpodautoscaler_spec_min_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + < + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + changes(kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 + for: {{ dig "KubeHpaReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaMaxedOut | default false) }} + - alert: KubeHpaMaxedOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has been running at max replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpamaxedout + summary: HPA is running at max replicas + expr: |- + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + == + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeHpaMaxedOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaMaxedOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml new file mode 100644 index 0000000000..1d32f9bbad --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -0,0 +1,282 @@ +{{- /* +Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesResources }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-resources" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-resources + rules: +{{- if not (.Values.defaultRules.disabled.KubeCPUOvercommit | default false) }} + - alert: KubeCPUOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Pods by {{`{{`}} $value {{`}}`}} CPU shares and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(namespace_cpu:kube_pod_container_resource_requests:sum{job="{{ $kubeStateMetricsJob }}",}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeCPUOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryOvercommit | default false) }} + - alert: KubeMemoryOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeMemoryOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeCPUQuotaOvercommit | default false) }} + - alert: KubeCPUQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuquotaovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(cpu|requests.cpu)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="cpu", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeCPUQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryQuotaOvercommit | default false) }} + - alert: KubeMemoryQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryquotaovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(memory|requests.memory)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeMemoryQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaAlmostFull | default false) }} + - alert: KubeQuotaAlmostFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaalmostfull + summary: Namespace quota is going to be full. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 0.9 < 1 + for: {{ dig "KubeQuotaAlmostFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaAlmostFull" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaFullyUsed | default false) }} + - alert: KubeQuotaFullyUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotafullyused + summary: Namespace quota is fully used. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + == 1 + for: {{ dig "KubeQuotaFullyUsed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaFullyUsed" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaExceeded | default false) }} + - alert: KubeQuotaExceeded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaexceeded + summary: Namespace quota has exceeded the limits. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 1 + for: {{ dig "KubeQuotaExceeded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaExceeded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.CPUThrottlingHigh | default false) }} + - alert: CPUThrottlingHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.pod {{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/cputhrottlinghigh + summary: Processes experience elevated CPU throttling. + expr: |- + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + / + sum(increase(container_cpu_cfs_periods_total{}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + > ( 25 / 100 ) + for: {{ dig "CPUThrottlingHigh" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "CPUThrottlingHigh" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml new file mode 100644 index 0000000000..b988445653 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -0,0 +1,216 @@ +{{- /* +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-storage + rules: +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + < 0.03 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} only has {{`{{`}} $value | humanizePercentage {{`}}`}} free inodes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to run out of inodes within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} of its inodes are free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeErrors | default false) }} + - alert: KubePersistentVolumeErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeerrors + summary: PersistentVolume is having issues with provisioning. + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="{{ $kubeStateMetricsJob }}"} > 0 + for: {{ dig "KubePersistentVolumeErrors" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml new file mode 100644 index 0000000000..af34a23f88 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -0,0 +1,193 @@ +{{- /* +Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-apiserver" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-apiserver + rules: +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIErrors | default false) }} + - alert: KubeAggregatedAPIErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapierrors + summary: Kubernetes aggregated API has reported errors. + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4 + labels: + severity: {{ dig "KubeAggregatedAPIErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIDown | default false) }} + - alert: KubeAggregatedAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapidown + summary: Kubernetes aggregated API is down. + expr: (1 - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85 + for: {{ dig "KubeAggregatedAPIDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAggregatedAPIDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if .Values.kubeApiServer.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeAPIDown | default false) }} + - alert: KubeAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: KubeAPI has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapidown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="apiserver"} == 1) + for: {{ dig "KubeAPIDown" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPIDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPITerminatedRequests | default false) }} + - alert: KubeAPITerminatedRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapiterminatedrequests + summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 + for: {{ dig "KubeAPITerminatedRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPITerminatedRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml new file mode 100644 index 0000000000..205bd59800 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeControllerManager }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-controller-manager" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-controller-manager + rules: +{{- if not (.Values.defaultRules.disabled.KubeControllerManagerDown | default false) }} + - alert: KubeControllerManagerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager | indent 8 }} +{{- end }} + description: KubeControllerManager has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontrollermanagerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeControllerManager.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeControllerManagerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} + diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml new file mode 100644 index 0000000000..66b1d62001 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml @@ -0,0 +1,56 @@ +{{- /* +Generated from 'kubernetes-system-kube-proxy' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeProxy }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kube-proxy" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kube-proxy + rules: +{{- if not (.Values.defaultRules.disabled.KubeProxyDown | default false) }} + - alert: KubeProxyDown + annotations: + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + description: KubeProxy has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeproxydown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeProxy.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeProxyDown" "labelsSeverity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml new file mode 100644 index 0000000000..2a55676735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -0,0 +1,379 @@ +{{- /* +Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kubelet" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kubelet + rules: +{{- if not (.Values.defaultRules.disabled.KubeNodeNotReady | default false) }} + - alert: KubeNodeNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than 15 minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodenotready + summary: Node is not ready. + expr: kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",condition="Ready",status="true"} == 0 + for: {{ dig "KubeNodeNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeUnreachable | default false) }} + - alert: KubeNodeUnreachable + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} is unreachable and some workloads may be rescheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodeunreachable + summary: Node is unreachable. + expr: (kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1 + for: {{ dig "KubeNodeUnreachable" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeUnreachable" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletTooManyPods | default false) }} + - alert: KubeletTooManyPods + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubelettoomanypods + summary: Kubelet is running at capacity. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + (kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}",phase="Running"} == 1) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) (1, kube_pod_info{job="{{ $kubeStateMetricsJob }}"}) + ) + / + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + kube_node_status_capacity{job="{{ $kubeStateMetricsJob }}",resource="pods"} != 1 + ) > 0.95 + for: {{ dig "KubeletTooManyPods" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletTooManyPods" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeReadinessFlapping | default false) }} + - alert: KubeNodeReadinessFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodereadinessflapping + summary: Node readiness status is flapping. + expr: sum(changes(kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",status="true",condition="Ready"}[15m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) > 2 + for: {{ dig "KubeNodeReadinessFlapping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeReadinessFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPlegDurationHigh | default false) }} + - alert: KubeletPlegDurationHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletplegdurationhigh + summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist. + expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10 + for: {{ dig "KubeletPlegDurationHigh" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletPlegDurationHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPodStartUpLatencyHigh | default false) }} + - alert: KubeletPodStartUpLatencyHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletpodstartuplatencyhigh + summary: Kubelet Pod startup latency is too high. + expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} > 60 + for: 15m + labels: + severity: {{ dig "KubeletPodStartUpLatencyHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateRenewalErrors | default false) }} + - alert: KubeletClientCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its client certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificaterenewalerrors + summary: Kubelet has failed to renew its client certificate. + expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletClientCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletClientCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateRenewalErrors | default false) }} + - alert: KubeletServerCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its server certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificaterenewalerrors + summary: Kubelet has failed to renew its server certificate. + expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletServerCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletServerCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if (include "exporter.kubelet.enabled" .)}} +{{- if not (.Values.defaultRules.disabled.KubeletDown | default false) }} + - alert: KubeletDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} == 1) + for: 15m + labels: + severity: {{ dig "KubeletDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml new file mode 100644 index 0000000000..9890b1c959 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -0,0 +1,54 @@ +{{- /* +Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-scheduler" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-scheduler + rules: +{{- if .Values.kubeScheduler.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeSchedulerDown | default false) }} + - alert: KubeSchedulerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting | indent 8 }} +{{- end }} + description: KubeScheduler has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeschedulerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeScheduler.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeSchedulerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml new file mode 100644 index 0000000000..621326d0ad --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -0,0 +1,87 @@ +{{- /* +Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system + rules: +{{- if not (.Values.defaultRules.disabled.KubeVersionMismatch | default false) }} + - alert: KubeVersionMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeversionmismatch + summary: Different semantic versions of Kubernetes components running. + expr: count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 + for: {{ dig "KubeVersionMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeVersionMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientErrors | default false) }} + - alert: KubeClientErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} $value | humanizePercentage {{`}}`}} errors.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclienterrors + summary: Kubernetes API server client is experiencing errors. + expr: |- + (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace) + / + sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace)) + > 0.01 + for: {{ dig "KubeClientErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml new file mode 100644 index 0000000000..5d4711ae08 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml @@ -0,0 +1,188 @@ +{{- /* +Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter.rules + rules: + - expr: |- + count without (cpu, mode) ( + node_cpu_seconds_total{job="node-exporter",mode="idle"} + ) + record: instance:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg without (cpu) ( + sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m])) + ) + record: instance:node_cpu_utilisation:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + ( + node_load1{job="node-exporter"} + / + instance:node_num_cpu:sum{job="node-exporter"} + ) + record: instance:node_load1_per_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + node_memory_MemAvailable_bytes{job="node-exporter"} + or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + + node_memory_Cached_bytes{job="node-exporter"} + + + node_memory_MemFree_bytes{job="node-exporter"} + + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) + / + node_memory_MemTotal_bytes{job="node-exporter"} + ) + record: instance:node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) + record: instance:node_vmstat_pgmajfault:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_weighted_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml new file mode 100644 index 0000000000..14738cedfa --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml @@ -0,0 +1,801 @@ +{{- /* +Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter + rules: +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 24 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 4 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 5% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 3% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 24 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 4 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 5% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 3% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkReceiveErrs | default false) }} + - alert: NodeNetworkReceiveErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworkreceiveerrs + summary: Network interface is reporting many receive errors. + expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkReceiveErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkReceiveErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkTransmitErrs | default false) }} + - alert: NodeNetworkTransmitErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworktransmiterrs + summary: Network interface is reporting many transmit errors. + expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkTransmitErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkTransmitErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeHighNumberConntrackEntriesUsed | default false) }} + - alert: NodeHighNumberConntrackEntriesUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodehighnumberconntrackentriesused + summary: Number of conntrack are getting close to the limit. + expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75 + labels: + severity: {{ dig "NodeHighNumberConntrackEntriesUsed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeTextFileCollectorScrapeError | default false) }} + - alert: NodeTextFileCollectorScrapeError + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodetextfilecollectorscrapeerror + summary: Node Exporter text file collector failed to scrape. + expr: node_textfile_scrape_error{job="node-exporter"} == 1 + labels: + severity: {{ dig "NodeTextFileCollectorScrapeError" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockSkewDetected | default false) }} + - alert: NodeClockSkewDetected + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclockskewdetected + summary: Clock skew detected. + expr: |- + ( + node_timex_offset_seconds{job="node-exporter"} > 0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0 + ) + or + ( + node_timex_offset_seconds{job="node-exporter"} < -0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0 + ) + for: {{ dig "NodeClockSkewDetected" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockSkewDetected" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockNotSynchronising | default false) }} + - alert: NodeClockNotSynchronising + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclocknotsynchronising + summary: Clock not synchronising. + expr: |- + min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0 + and + node_timex_maxerror_seconds{job="node-exporter"} >= 16 + for: {{ dig "NodeClockNotSynchronising" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockNotSynchronising" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDegraded | default false) }} + - alert: NodeRAIDDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: RAID array '{{`{{`}} $labels.device {{`}}`}}' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddegraded + summary: RAID Array is degraded. + expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0 + for: {{ dig "NodeRAIDDegraded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeRAIDDegraded" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDiskFailure | default false) }} + - alert: NodeRAIDDiskFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddiskfailure + summary: Failed device in RAID array. + expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0 + labels: + severity: {{ dig "NodeRAIDDiskFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeCPUHighUsage | default false) }} + - alert: NodeCPUHighUsage + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodecpuhighusage + summary: High CPU usage. + expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!="idle"}[2m]))) * 100 > 90 + for: {{ dig "NodeCPUHighUsage" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeCPUHighUsage" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemSaturation | default false) }} + - alert: NodeSystemSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This might indicate this instance resources saturation and can cause it becoming unresponsive. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemsaturation + summary: System saturated, load per core is very high. + expr: |- + node_load1{job="node-exporter"} + / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2 + for: {{ dig "NodeSystemSaturation" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryMajorPagesFaults | default false) }} + - alert: NodeMemoryMajorPagesFaults + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + Please check that there is enough memory available at this instance. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememorymajorpagesfaults + summary: Memory major page faults are occurring at very high rate. + expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500 + for: {{ dig "NodeMemoryMajorPagesFaults" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryMajorPagesFaults" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryHighUtilization | default false) }} + - alert: NodeMemoryHighUtilization + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememoryhighutilization + summary: Host is running out of memory. + expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90 + for: {{ dig "NodeMemoryHighUtilization" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryHighUtilization" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeDiskIOSaturation | default false) }} + - alert: NodeDiskIOSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 30 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This symptom might indicate disk saturation. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodediskiosaturation + summary: Disk IO queue is high. + expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10 + for: {{ dig "NodeDiskIOSaturation" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeDiskIOSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemdServiceFailed | default false) }} + - alert: NodeSystemdServiceFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemdservicefailed + summary: Systemd service has entered failed state. + expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1 + for: {{ dig "NodeSystemdServiceFailed" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemdServiceFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeBondingDegraded | default false) }} + - alert: NodeBondingDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodebondingdegraded + summary: Bonding interface is degraded + expr: (node_bonding_slaves - node_bonding_active) != 0 + for: {{ dig "NodeBondingDegraded" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeBondingDegraded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml new file mode 100644 index 0000000000..8dc60ef66b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.network }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-network" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-network + rules: +{{- if not (.Values.defaultRules.disabled.NodeNetworkInterfaceFlapping | default false) }} + - alert: NodeNetworkInterfaceFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.network }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.network | indent 8 }} +{{- end }} + description: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing its up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/nodenetworkinterfaceflapping + summary: Network interface is often changing its status + expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 + for: {{ dig "NodeNetworkInterfaceFlapping" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkInterfaceFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml new file mode 100644 index 0000000000..e2911b905e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml @@ -0,0 +1,109 @@ +{{- /* +Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.node }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node.rules + rules: + - expr: |- + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node, namespace, pod) ( + label_replace(kube_pod_info{job="{{ $kubeStateMetricsJob }}",node!=""}, "pod", "$1", "pod", "(.*)") + )) + record: 'node_namespace_pod:kube_pod_info:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + node_cpu_seconds_total{mode="idle",job="node-exporter"} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, node_namespace_pod:kube_pod_info:) + ) + record: node:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum( + node_memory_MemAvailable_bytes{job="node-exporter"} or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + node_memory_Cached_bytes{job="node-exporter"} + + node_memory_MemFree_bytes{job="node-exporter"} + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + record: :node_memory_MemAvailable_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + sum without (mode) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m]) + ) + ) + record: node:node_cpu_utilization:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + node:node_cpu_utilization:ratio_rate5m + ) + record: cluster:node_cpu:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml new file mode 100644 index 0000000000..1f288dffe3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml @@ -0,0 +1,253 @@ +{{- /* +Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheusOperator }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus-operator" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus-operator + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorListErrors | default false) }} + - alert: PrometheusOperatorListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing List operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorlisterrors + summary: Errors while performing list operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m]))) > 0.4 + for: {{ dig "PrometheusOperatorListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorListErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorWatchErrors | default false) }} + - alert: PrometheusOperatorWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing watch operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorwatcherrors + summary: Errors while performing watch operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.4 + for: {{ dig "PrometheusOperatorWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorWatchErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorSyncFailed | default false) }} + - alert: PrometheusOperatorSyncFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Controller {{`{{`}} $labels.controller {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} namespace fails to reconcile {{`{{`}} $value {{`}}`}} objects. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorsyncfailed + summary: Last controller reconciliation failed + expr: min_over_time(prometheus_operator_syncs{status="failed",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorSyncFailed" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorSyncFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorReconcileErrors | default false) }} + - alert: PrometheusOperatorReconcileErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of reconciling operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorreconcileerrors + summary: Errors while reconciling objects. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorReconcileErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorReconcileErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorStatusUpdateErrors | default false) }} + - alert: PrometheusOperatorStatusUpdateErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of status update operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorstatusupdateerrors + summary: Errors while updating objects status. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorStatusUpdateErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorStatusUpdateErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNodeLookupErrors | default false) }} + - alert: PrometheusOperatorNodeLookupErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while reconciling Prometheus in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornodelookuperrors + summary: Errors while reconciling Prometheus. + expr: rate(prometheus_operator_node_address_lookup_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: {{ dig "PrometheusOperatorNodeLookupErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNodeLookupErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNotReady | default false) }} + - alert: PrometheusOperatorNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace isn't ready to reconcile {{`{{`}} $labels.controller {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornotready + summary: Prometheus operator not ready + expr: min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) == 0) + for: {{ dig "PrometheusOperatorNotReady" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorRejectedResources | default false) }} + - alert: PrometheusOperatorRejectedResources + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace rejected {{`{{`}} printf "%0.0f" $value {{`}}`}} {{`{{`}} $labels.controller {{`}}`}}/{{`{{`}} $labels.resource {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorrejectedresources + summary: Resources rejected by Prometheus operator + expr: min_over_time(prometheus_operator_managed_resources{state="rejected",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorRejectedResources" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorRejectedResources" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml new file mode 100644 index 0000000000..9dfeb1f9db --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml @@ -0,0 +1,707 @@ +{{- /* +Generated from 'prometheus' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} +{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusBadConfig | default false) }} + - alert: PrometheusBadConfig + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to reload its configuration. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusbadconfig + summary: Failed Prometheus configuration reload. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "PrometheusBadConfig" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusBadConfig" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusSDRefreshFailure | default false) }} + - alert: PrometheusSDRefreshFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to refresh SD with mechanism {{`{{`}}$labels.mechanism{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheussdrefreshfailure + summary: Failed Prometheus SD refresh. + expr: increase(prometheus_sd_refresh_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[10m]) > 0 + for: {{ dig "PrometheusSDRefreshFailure" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusSDRefreshFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotificationQueueRunningFull | default false) }} + - alert: PrometheusNotificationQueueRunningFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Alert notification queue of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is running full. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotificationqueuerunningfull + summary: Prometheus alert notification queue predicted to run full in less than 30m. + expr: |- + # Without min_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) + > + min_over_time(prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusNotificationQueueRunningFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotificationQueueRunningFull" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToSomeAlertmanagers | default false) }} + - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.alertmanager{{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstosomealertmanagers + summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager. + expr: |- + ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + * 100 + > 1 + for: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotConnectedToAlertmanagers | default false) }} + - alert: PrometheusNotConnectedToAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not connected to any Alertmanagers. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotconnectedtoalertmanagers + summary: Prometheus is not connected to any Alertmanagers. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) < 1 + for: {{ dig "PrometheusNotConnectedToAlertmanagers" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotConnectedToAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBReloadsFailing | default false) }} + - alert: PrometheusTSDBReloadsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} reload failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbreloadsfailing + summary: Prometheus has issues reloading blocks from disk. + expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBReloadsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBReloadsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBCompactionsFailing | default false) }} + - alert: PrometheusTSDBCompactionsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbcompactionsfailing + summary: Prometheus has issues compacting blocks. + expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBCompactionsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBCompactionsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotIngestingSamples | default false) }} + - alert: PrometheusNotIngestingSamples + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not ingesting samples. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotingestingsamples + summary: Prometheus is not ingesting samples. + expr: |- + ( + sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) <= 0 + and + ( + sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + or + sum without(rule_group) (prometheus_rule_group_rules{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + ) + ) + for: {{ dig "PrometheusNotIngestingSamples" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotIngestingSamples" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusDuplicateTimestamps | default false) }} + - alert: PrometheusDuplicateTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with different values but duplicated timestamp. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusduplicatetimestamps + summary: Prometheus is dropping samples with duplicate timestamps. + expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusDuplicateTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusDuplicateTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOutOfOrderTimestamps | default false) }} + - alert: PrometheusOutOfOrderTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with timestamps arriving out of order. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusoutofordertimestamps + summary: Prometheus drops samples with out-of-order timestamps. + expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOutOfOrderTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOutOfOrderTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteStorageFailures | default false) }} + - alert: PrometheusRemoteStorageFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotestoragefailures + summary: Prometheus fails to send samples to remote storage. + expr: |- + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + / + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + + + (rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + ) + ) + * 100 + > 1 + for: {{ dig "PrometheusRemoteStorageFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteStorageFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteBehind | default false) }} + - alert: PrometheusRemoteWriteBehind + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritebehind + summary: Prometheus remote write is behind. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + - ignoring(remote_name, url) group_right + max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 120 + for: {{ dig "PrometheusRemoteWriteBehind" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteBehind" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteDesiredShards | default false) }} + - alert: PrometheusRemoteWriteDesiredShards + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritedesiredshards + summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_shards_desired{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + > + max_over_time(prometheus_remote_storage_shards_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusRemoteWriteDesiredShards" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteDesiredShards" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRuleFailures | default false) }} + - alert: PrometheusRuleFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to evaluate {{`{{`}} printf "%.0f" $value {{`}}`}} rules in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusrulefailures + summary: Prometheus is failing rule evaluations. + expr: increase(prometheus_rule_evaluation_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusRuleFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRuleFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusMissingRuleEvaluations | default false) }} + - alert: PrometheusMissingRuleEvaluations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has missed {{`{{`}} printf "%.0f" $value {{`}}`}} rule group evaluations in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusmissingruleevaluations + summary: Prometheus is missing rule evaluations due to slow rule group evaluation. + expr: increase(prometheus_rule_group_iterations_missed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusMissingRuleEvaluations" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusMissingRuleEvaluations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetLimitHit | default false) }} + - alert: PrometheusTargetLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because the number of targets exceeded the configured target_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetlimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit. + expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusTargetLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusLabelLimitHit | default false) }} + - alert: PrometheusLabelLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuslabellimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit. + expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusLabelLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusLabelLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeBodySizeLimitHit | default false) }} + - alert: PrometheusScrapeBodySizeLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured body_size_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapebodysizelimithit + summary: Prometheus has dropped some targets that exceeded body size limit. + expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeBodySizeLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeBodySizeLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeSampleLimitHit | default false) }} + - alert: PrometheusScrapeSampleLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured sample_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapesamplelimithit + summary: Prometheus has failed scrapes that have exceeded the configured sample limit. + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeSampleLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeSampleLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetSyncFailure | default false) }} + - alert: PrometheusTargetSyncFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.0f" $value {{`}}`}} targets in Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} have failed to sync because invalid configuration was supplied.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetsyncfailure + summary: Prometheus has failed to sync targets. + expr: increase(prometheus_target_sync_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[30m]) > 0 + for: {{ dig "PrometheusTargetSyncFailure" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetSyncFailure" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusHighQueryLoad | default false) }} + - alert: PrometheusHighQueryLoad + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} query API has less than 20% available capacity in its query engine for the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheushighqueryload + summary: Prometheus is reaching its maximum capacity serving concurrent requests. + expr: avg_over_time(prometheus_engine_queries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.8 + for: {{ dig "PrometheusHighQueryLoad" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusHighQueryLoad" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToAnyAlertmanager | default false) }} + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstoanyalertmanager + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without (alertmanager) ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + ) + * 100 + > 3 + for: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml new file mode 100644 index 0000000000..7c25553861 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml @@ -0,0 +1,301 @@ +{{- /* +Generated from 'windows.node.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.node.rules + rules: + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + windows_system_system_up_time{job="windows-exporter"} + ) + record: node:windows_node:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, core) ( + windows_cpu_time_total{job="windows-exporter"} + )) + record: node:windows_node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m])) + record: :windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m]) + ) + record: node:windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"}) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: ':windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"} + windows_memory_cache_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemFreeCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: (windows_memory_cache_bytes{job="windows-exporter"} + windows_memory_modified_page_list_bytes{job="windows-exporter"} + windows_memory_standby_cache_core_bytes{job="windows-exporter"} + windows_memory_standby_cache_normal_priority_bytes{job="windows-exporter"} + windows_memory_standby_cache_reserve_bytes{job="windows-exporter"}) + record: node:windows_node_memory_totalCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemTotal_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (windows_memory_available_bytes{job="windows-exporter"}) + ) + record: node:windows_node_memory_bytes_available:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + windows_os_visible_memory_bytes{job="windows-exporter"} + ) + record: node:windows_node_memory_bytes_total:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + (node:windows_node_memory_bytes_total:sum - node:windows_node_memory_bytes_available:sum) + / + scalar(sum(node:windows_node_memory_bytes_total:sum)) + record: node:windows_node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - (node:windows_node_memory_bytes_available:sum / node:windows_node_memory_bytes_total:sum) + record: 'node:windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: irate(windows_memory_swap_page_operations_total{job="windows-exporter"}[5m]) + record: node:windows_node_memory_swap_io_pages:irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m]) + ) + record: :windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,instance,volume)( + (windows_logical_disk_size_bytes{job="windows-exporter"} + - windows_logical_disk_free_bytes{job="windows-exporter"}) + / windows_logical_disk_size_bytes{job="windows-exporter"} + ) + record: 'node:windows_node_filesystem_usage:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, volume) (windows_logical_disk_free_bytes{job="windows-exporter"} / windows_logical_disk_size_bytes{job="windows-exporter"}) + record: 'node:windows_node_filesystem_avail:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + record: :windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m])) + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + record: :windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m]) + + irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml new file mode 100644 index 0000000000..86340b5c05 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml @@ -0,0 +1,158 @@ +{{- /* +Generated from 'windows.pod.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.pod.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.pod.rules + rules: + - expr: windows_container_available{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_pod_container_available + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_cpu_usage_seconds_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_total_runtime + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_commit_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_memory_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_private_working_set_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_private_working_set_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_receive_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_received_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_transmit_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_transmitted_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + rate(windows_container_total_runtime{}[5m]) + ) + record: namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/secret.yaml new file mode 100644 index 0000000000..e4a1e73c7b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/secret.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.objectStorageConfig}} +{{- if and .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret (not .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + object-storage-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/service.yaml new file mode 100644 index 0000000000..d61b9d6ef3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/service.yaml @@ -0,0 +1,80 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + self-monitor: {{ .Values.prometheus.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.service.labels }} +{{ toYaml .Values.prometheus.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheus.service.clusterIP }} + clusterIP: {{ .Values.prometheus.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.service.nodePort }} + {{- end }} + port: {{ .Values.prometheus.service.port }} + targetPort: {{ .Values.prometheus.service.targetPort }} + - name: reloader-web + {{- if semverCompare "> 1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: {{ .Values.prometheus.service.reloaderWebPort }} + targetPort: reloader-web + {{- if .Values.prometheus.thanosIngress.enabled }} + - name: grpc + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosIngress.nodePort }} + {{- end }} + port: {{ .Values.prometheus.thanosIngress.servicePort }} + targetPort: {{ .Values.prometheus.thanosIngress.servicePort }} + {{- end }} +{{- if .Values.prometheus.service.additionalPorts }} +{{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} +{{- end }} + publishNotReadyAddresses: {{ .Values.prometheus.service.publishNotReadyAddresses }} + selector: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- if .Values.prometheus.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.prometheus.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.prometheus.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.prometheus.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml new file mode 100644 index 0000000000..15b89c8c23 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-discovery + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosService.labels }} +{{ toYaml .Values.prometheus.thanosService.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosService.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosService.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosService.type }} + clusterIP: {{ .Values.prometheus.thanosService.clusterIP }} +{{- if ne .Values.prometheus.thanosService.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosService.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosService.portName }} + port: {{ .Values.prometheus.thanosService.port }} + targetPort: {{ .Values.prometheus.thanosService.targetPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosService.httpPortName }} + port: {{ .Values.prometheus.thanosService.httpPort }} + targetPort: {{ .Values.prometheus.thanosService.targetHttpPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml new file mode 100644 index 0000000000..453eed7f1b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosServiceExternal.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-external + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosServiceExternal.labels }} +{{ toYaml .Values.prometheus.thanosServiceExternal.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosServiceExternal.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosServiceExternal.type }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.thanosServiceExternal.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.thanosServiceExternal.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosServiceExternal.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosServiceExternal.portName }} + port: {{ .Values.prometheus.thanosServiceExternal.port }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosServiceExternal.httpPortName }} + port: {{ .Values.prometheus.thanosServiceExternal.httpPort }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetHttpPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceaccount.yaml new file mode 100644 index 0000000000..e97b989bbd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.prometheus.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.prometheus.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitor.yaml new file mode 100644 index 0000000000..a36f3e33ca --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitor.yaml @@ -0,0 +1,97 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.serviceMonitor.bearerTokenFile }} + {{- end }} + path: "{{ trimSuffix "/" .Values.prometheus.prometheusSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + - port: reloader-web + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "/metrics" + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.prometheus.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.prometheus.serviceMonitor.interval .interval }} + interval: {{ default $.Values.prometheus.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.prometheus.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.prometheus.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml new file mode 100644 index 0000000000..0f70aabb58 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.prometheus.thanosService.enabled .Values.prometheus.thanosServiceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-sidecar + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-sidecar +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.thanosServiceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.thanosServiceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.thanosService.httpPortName }} + {{- if .Values.prometheus.thanosServiceMonitor.interval }} + interval: {{ .Values.prometheus.thanosServiceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.scheme }} + scheme: {{ .Values.prometheus.thanosServiceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.prometheus.thanosServiceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + {{- end }} + path: "/metrics" + metricRelabelings: + {{- if .Values.prometheus.thanosServiceMonitor.metricRelabelings}} + {{ tpl (toYaml .Values.prometheus.thanosServiceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheus.thanosServiceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.thanosServiceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitors.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitors.yaml new file mode 100644 index 0000000000..a7a301babc --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitors.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalServiceMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalServiceMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + endpoints: +{{ toYaml .endpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .targetLabels }} + targetLabels: +{{ toYaml .targetLabels | indent 8 }} + {{- end }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .metricRelabelings }} + metricRelabelings: +{{ toYaml .metricRelabelings | indent 8 }} + {{- end }} + {{- if .relabelings }} + relabelings: +{{ toYaml .relabelings | indent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceperreplica.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceperreplica.yaml new file mode 100644 index 0000000000..4bc7f7b869 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceperreplica.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $serviceValues := .Values.prometheus.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.prometheus.prometheusSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + {{- if $.Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + statefulset.kubernetes.io/pod-name: prom-agent-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- else }} + app.kubernetes.io/name: prometheus + statefulset.kubernetes.io/pod-name: prometheus-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" $ }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/clusterrole.yaml new file mode 100644 index 0000000000..56ca9f5eae --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/clusterrole.yaml @@ -0,0 +1,135 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-admin + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-edit + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-ui-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" +{{- if .Values.grafana.enabled }} + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" +{{- end }} + verbs: + - 'get' + - 'create' +- apiGroups: + - "" + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- if .Values.grafana.enabled }} + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +{{- end }} + resources: + - endpoints + verbs: + - list +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/config-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/config-role.yaml new file mode 100644 index 0000000000..f48ffc827e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/config-role.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-admin + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-edit + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-view + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml new file mode 100644 index 0000000000..d2f81976a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create .Values.grafana.enabled }} +{{- if .Values.grafana.defaultDashboardsEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-admin + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-edit + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-view + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml new file mode 100644 index 0000000000..7b51a0bf7a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.ingressNginx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "ingress-nginx" | trunc 63 | trimSuffix "-" }} + {{- if .Values.grafana.sidecar.dashboards.annotations }} + annotations: {{ toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} + {{- end }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/ingress-nginx/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml new file mode 100644 index 0000000000..d73b257451 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-cluster + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/cluster/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml new file mode 100644 index 0000000000..8865efa932 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-home + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/home/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml new file mode 100644 index 0000000000..9b05cea2e8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-fleet-dashboards + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/fleet/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml new file mode 100644 index 0000000000..2afae10ef7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml @@ -0,0 +1,31 @@ +{{- $files := (.Files.Glob "files/rancher/k8s/*").AsConfig }} +{{- $filesDict := (fromYaml $files) }} +{{- if not (include "exporter.kubeEtcd.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-etcd-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-etcd.json") -}} +{{- end }} +{{- if not (include "exporter.kubeControllerManager.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-k8s-components-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-k8s-components.json") -}} +{{- else }} +{{- $_ := (set $filesDict "rancher-k8s-components-nodes.json" (get $filesDict "rancher-k8s-components-nodes.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- $_ := (set $filesDict "rancher-k8s-components.json" (get $filesDict "rancher-k8s-components.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- end }} +{{ $files = (toYaml $filesDict) }} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-k8s + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ $files | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml new file mode 100644 index 0000000000..172c36e9d1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-nodes + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/nodes/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml new file mode 100644 index 0000000000..19836ec4e4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml @@ -0,0 +1,18 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.rancherMonitoring.enabled $selector }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-performance-debugging + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/performance/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml new file mode 100644 index 0000000000..940f18869b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-pods + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/pods/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml new file mode 100644 index 0000000000..d146dacdd0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-workloads + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/workloads/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml new file mode 100644 index 0000000000..90d24c2061 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml @@ -0,0 +1,53 @@ +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-fleet-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: fleet + selector: + matchLabels: + app: fleet-controller +{{- end }} +--- +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-gitops-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: gitops + selector: + matchLabels: + app: gitjob +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml new file mode 100644 index 0000000000..9d3e515f39 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rke2IngressNginx.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + annotations: + np.rke2.io/ingress: resolved + name: rke2-ingress-network-policy + namespace: {{ include "rke2-ingress-nginx.namespace" . }} +spec: + ingress: + - ports: + - port: {{ .Values.rke2IngressNginx.metricsPort }} + protocol: TCP + podSelector: + matchLabels: + app.kubernetes.io/name: rke2-ingress-nginx + policyTypes: + - Ingress +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml new file mode 100644 index 0000000000..53a9ad6897 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml @@ -0,0 +1,27 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + jobLabel: ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: {{ .Values.ingressNginx.namespace }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.ingressNginx.service.port }} + protocol: TCP + targetPort: {{ .Values.ingressNginx.service.targetPort }} + selector: + {{- if .Values.ingressNginx.service.selector }} +{{ toYaml .Values.ingressNginx.service.selector | indent 4 }} + {{- else }} + app: ingress-nginx + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml new file mode 100644 index 0000000000..b0f92e63b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml @@ -0,0 +1,49 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + namespace: {{ .Values.ingressNginx.namespace }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Values.ingressNginx.namespace }} + endpoints: + - port: http-metrics + {{- if .Values.ingressNginx.serviceMonitor.interval}} + interval: {{ .Values.ingressNginx.serviceMonitor.interval }} + {{- end }} + {{- if .Values.ingressNginx.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.ingressNginx.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.ingressNginx.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.ingressNginx.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.ingressNginx.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.ingressNginx.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml new file mode 100644 index 0000000000..1fba8f23f7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.rancherMonitoring.enabled $selector }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: rancher + namespace: cattle-system +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + port: http + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + serverName: rancher + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: rancher +{{- if .Values.rancherMonitoring.namespaceSelector }} + namespaceSelector: {{ .Values.rancherMonitoring.namespaceSelector | toYaml | nindent 4 }} +{{- end }} + selector: {{ include "rancher.serviceMonitor.selector" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +rules: +- apiGroups: + - management.cattle.io + resources: + - ranchermetrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/hardened.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/hardened.yaml new file mode 100644 index 0000000000..f9a66151ee --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/hardened.yaml @@ -0,0 +1,147 @@ +{{- $namespaces := dict "_0" .Release.Namespace -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) -}} +{{- $_ := set $namespaces "_1" .Values.grafana.defaultDashboards.namespace -}} +{{- end -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa + spec: + serviceAccountName: {{ .Chart.Name }}-patch-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + {{- range $_, $ns := $namespaces }} + - name: patch-sa-{{ $ns }} + image: {{ template "system_default_registry" $ }}{{ $.Values.global.kubectl.repository }}:{{ $.Values.global.kubectl.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", "{{ $ns }}"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +rules: +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: ['get', 'patch'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-patch-sa +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-patch-sa +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- range $_, $ns := $namespaces }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ $ns }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +{{- end }} +{{- end }} +--- +{{- if .Values.hardened.k3s.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rancher-monitoring-coredns-allow-all + namespace: kube-system +spec: + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + podSelector: + matchLabels: + k8s-app: kube-dns +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml new file mode 100644 index 0000000000..53cb898214 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "0" +data: +{{ (.Files.Glob "files/upgrade/scripts/*").AsConfig | indent 2 }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml new file mode 100644 index 0000000000..8f2771740c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml @@ -0,0 +1,46 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "2" +spec: + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + spec: + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + securityContext: + runAsNonRoot: false + runAsUser: 0 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + - name: run-scripts + image: {{ template "system_default_registry" . }}{{ .Values.upgrade.image.repository }}:{{ .Values.upgrade.image.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: + - /bin/sh + - -c + - > + for s in $(find /etc/scripts -type f); do + echo "Running $s..."; + cat $s | bash + done; + volumeMounts: + - name: upgrade + mountPath: /etc/scripts + volumes: + - name: upgrade + configMap: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml new file mode 100644 index 0000000000..e929a19925 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml @@ -0,0 +1,131 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded + "helm.sh/hook-weight": "1" +rules: +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - 'list' + - 'delete' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/extrasecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/extrasecret.yaml new file mode 100644 index 0000000000..587fca2dca --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.thanosRuler.extraSecret.data -}} +{{- $secretName := printf "%s-extra" (include "kube-prometheus-stack.thanosRuler.name" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.thanosRuler.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.extraSecret.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.thanosRuler.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ingress.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ingress.yaml new file mode 100644 index 0000000000..e245ad448e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.ingress.enabled }} +{{- $pathType := .Values.thanosRuler.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := include "kube-prometheus-stack.thanosRuler.name" . }} +{{- $servicePort := .Values.thanosRuler.service.port -}} +{{- $routePrefix := list .Values.thanosRuler.thanosRulerSpec.routePrefix }} +{{- $paths := .Values.thanosRuler.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.thanosRuler.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- if .Values.thanosRuler.ingress.labels }} +{{ toYaml .Values.thanosRuler.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.thanosRuler.ingress.ingressClassName }} + ingressClassName: {{ .Values.thanosRuler.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.thanosRuler.ingress.hosts }} + {{- range $host := .Values.thanosRuler.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.thanosRuler.ingress.tls }} + tls: +{{ tpl (toYaml .Values.thanosRuler.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml new file mode 100644 index 0000000000..83e54edf9b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.thanosRuler.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.thanosRuler.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ruler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ruler.yaml new file mode 100644 index 0000000000..b281221563 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ruler.yaml @@ -0,0 +1,189 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ThanosRuler +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.thanosRulerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.thanosRuler.thanosRulerSpec.image.registry -}} + {{- if and .Values.thanosRuler.thanosRulerSpec.image.tag .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}" + {{- end }} + {{- if .Values.thanosRuler.thanosRulerSpec.image.sha }} + sha: {{ .Values.thanosRuler.thanosRulerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.thanosRuler.thanosRulerSpec.replicas }} + listenLocal: {{ .Values.thanosRuler.thanosRulerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} +{{- if .Values.thanosRuler.thanosRulerSpec.externalPrefix }} + externalPrefix: "{{ tpl .Values.thanosRuler.thanosRulerSpec.externalPrefix . }}" +{{- else if and .Values.thanosRuler.ingress.enabled .Values.thanosRuler.ingress.hosts }} + externalPrefix: "http://{{ tpl (index .Values.thanosRuler.ingress.hosts 0) . }}{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- else }} + externalPrefix: http://{{ template "kube-prometheus-stack.thanosRuler.name" . }}.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.thanosRuler.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.additionalArgs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.nodeSelector }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.thanosRuler.thanosRulerSpec.paused }} + logFormat: {{ .Values.thanosRuler.thanosRulerSpec.logFormat | quote }} + logLevel: {{ .Values.thanosRuler.thanosRulerSpec.logLevel | quote }} + retention: {{ .Values.thanosRuler.thanosRulerSpec.retention | quote }} +{{- if .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} + evaluationInterval: {{ .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleSelector | indent 4) .}} +{{- else if .Values.thanosRuler.thanosRulerSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }} + alertQueryUrl: "{{ .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }}" +{{- end}} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl }} + alertmanagersUrl: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret }} + alertmanagersConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.secret }} + alertmanagersConfig: + key: alertmanager-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryEndpoints }} + queryEndpoints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.queryEndpoints | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret }} + queryConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.queryConfig.secret }} + queryConfig: + key: query-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.resources }} + resources: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.routePrefix }} + routePrefix: "{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.securityContext }} + securityContext: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.storage }} + storage: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.storage | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret }} + objectStorageConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.secret }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.labels }} + labels: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.thanosRuler.thanosRulerSpec.podAntiAffinity .Values.thanosRuler.thanosRulerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.affinity }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- else if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.tolerations }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.containers }} + containers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.initContainers }} + initContainers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.priorityClassName }} + priorityClassName: {{.Values.thanosRuler.thanosRulerSpec.priorityClassName }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumes }} + volumes: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.thanosRuler.thanosRulerSpec.portName }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/secret.yaml new file mode 100644 index 0000000000..acab7fd9ae --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/secret.yaml @@ -0,0 +1,26 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + {{- with .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig }} + {{- if and .secret (not .existingSecret) }} + alertmanager-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.objectStorageConfig }} + {{- if and .secret (not .existingSecret) }} + object-storage-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.queryConfig }} + {{- if and .secret (not .existingSecret) }} + query-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/service.yaml new file mode 100644 index 0000000000..be0c844591 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/service.yaml @@ -0,0 +1,53 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.service.labels }} +{{ toYaml .Values.thanosRuler.service.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.service.clusterIP }} + clusterIP: {{ .Values.thanosRuler.service.clusterIP }} +{{- end }} +{{- if .Values.thanosRuler.service.externalIPs }} + externalIPs: +{{ toYaml .Values.thanosRuler.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.thanosRuler.service.loadBalancerIP }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.thanosRuler.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.thanosRuler.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.thanosRuler.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if eq .Values.thanosRuler.service.type "NodePort" }} + nodePort: {{ .Values.thanosRuler.service.nodePort }} + {{- end }} + port: {{ .Values.thanosRuler.service.port }} + targetPort: {{ .Values.thanosRuler.service.targetPort }} + protocol: TCP +{{- if .Values.thanosRuler.service.additionalPorts }} +{{ toYaml .Values.thanosRuler.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + type: "{{ .Values.thanosRuler.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/serviceaccount.yaml new file mode 100644 index 0000000000..b58f1cd4df --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/servicemonitor.yaml new file mode 100644 index 0000000000..b2b138b498 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/servicemonitor.yaml @@ -0,0 +1,82 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.thanosRuler.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.thanosRuler.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + release: {{ $.Release.Name | quote }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if .Values.thanosRuler.serviceMonitor.interval }} + interval: {{ .Values.thanosRuler.serviceMonitor.interval }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.thanosRuler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.scheme }} + scheme: {{ .Values.thanosRuler.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.thanosRuler.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.thanosRuler.thanosRulerSpec.routePrefix }}/metrics" + {{- if .Values.thanosRuler.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.thanosRuler.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.thanosRuler.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.thanosRuler.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.thanosRuler.serviceMonitor.interval .interval }} + interval: {{ default $.Values.thanosRuler.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.thanosRuler.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..6fcb8b3a69 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-install-crd.yaml @@ -0,0 +1,23 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/AlertmanagerConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/Alertmanager" false -}} +# {{- set $found "monitoring.coreos.com/v1/PodMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/Probe" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/PrometheusAgent" false -}} +# {{- set $found "monitoring.coreos.com/v1/Prometheus" false -}} +# {{- set $found "monitoring.coreos.com/v1/PrometheusRule" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/ScrapeConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/ThanosRuler" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/values.yaml new file mode 100644 index 0000000000..40f676556a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/values.yaml @@ -0,0 +1,5433 @@ +# Default values for kube-prometheus-stack. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Rancher Monitoring Configuration + +## Configuration for prometheus-adapter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter +## +prometheus-adapter: + enabled: true + prometheus: + # Change this if you change the namespaceOverride or nameOverride of prometheus-operator + url: http://rancher-monitoring-prometheus.cattle-monitoring-system.svc + port: 9090 + +## RKE PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rkeControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.23" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.23 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeEtcd: + enabled: false + metricsPort: 2379 + component: kube-etcd + clients: + port: 10014 + https: + enabled: true + certDir: /etc/kubernetes/ssl + certFile: kube-etcd-*.pem + keyFile: kube-etcd-*-key.pem + caCertFile: kube-ca.pem + seLinuxOptions: + # Gives rkeEtcd permissions to read files in /etc/kubernetes/* + # Type is defined in https://github.com/rancher/rancher-selinux + type: rke_kubereader_t + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeIngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + clients: + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + nodeSelector: + node-role.kubernetes.io/worker: "true" + +## k3s PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +k3sServer: + enabled: false + metricsPort: 10250 + component: k3s-server + clients: + port: 10013 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +hardened: + k3s: + networkPolicy: + enabled: true + +## KubeADM PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +kubeAdmControllerManager: + enabled: false + metricsPort: 10257 + component: kube-controller-manager + clients: + port: 10011 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + port: 10012 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmEtcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## rke2 PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rke2ControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Scheduler: + enabled: false + metricsPort: 10259 # default to secure port as of k8s >= 1.22 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Proxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2Etcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2IngressNginx: + enabled: false + networkPolicy: + enabled: false + metricsPort: 10254 + component: ingress-nginx + # in the RKE2 cluster, the ingress-nginx-controller is deployed + # as a non-hostNetwork workload starting at the following versions + # - >= v1.22.12+rke2r1 < 1.23.0-0 + # - >= v1.23.9+rke2r1 < 1.24.0-0 + # - >= v1.24.3+rke2r1 < 1.25.0-0 + # - >= v1.25.0+rke2r1 + # As a result we do not need clients and proxies as we can directly create + # a service that targets the workload with the given app name + namespaceOverride: kube-system + clients: + enabled: false + proxy: + enabled: false + service: + selector: + app.kubernetes.io/name: rke2-ingress-nginx + kubeVersionOverrides: + - constraint: "< 1.21.0-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a DaemonSet with 1 pod when RKE2 version is < 1.21.0-0 + deployment: + enabled: false + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.21.0-0 < 1.22.12-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.21.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.23.0-0 < v1.23.9-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.24.0-0 < v1.24.3-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + + + +## Additional PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## + +# hardenedKubelet can only be deployed if kubelet.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default kubelet option with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedKubelet: + enabled: false + metricsPort: 10250 + component: kubelet + clients: + port: 10015 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +# hardenedNodeExporter can only be deployed if nodeExporter.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default nodeExporter with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedNodeExporter: + enabled: false + metricsPort: 9796 + component: node-exporter + clients: + port: 10016 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## Upgrades +upgrade: + ## Run upgrade scripts before an upgrade or rollback via a Job hook + enabled: true + ## Image to use to run the scripts + image: + repository: rancher/shell + tag: v0.2.1 + +## Rancher Monitoring +## + +rancherMonitoring: + enabled: true + + ## A namespaceSelector to identify the namespace to find the Rancher deployment + ## + namespaceSelector: + matchNames: + - cattle-system + + ## A selector to identify the Rancher deployment + ## If not set, the chart will try to search for the Rancher deployment in the cattle-system namespace and infer the selector values from it + ## If the Rancher deployment does not exist, no resources will be deployed. + ## + selector: {} + +## Component scraping nginx-ingress-controller +## +ingressNginx: + enabled: false + + ## The namespace to search for your nginx-ingress-controller + ## + namespace: ingress-nginx + + service: + port: 9913 + targetPort: 10254 + # selector: + # app: ingress-nginx + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "30s" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +# Prometheus Operator Configuration + +## Provide a name in place of kube-prometheus-stack for `app:` labels +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +nameOverride: "rancher-monitoring" + +## Override the deployment namespace +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +namespaceOverride: "cattle-monitoring-system" + +## Provide a k8s version to auto dashboard import script example: kubeTargetVersionOverride: 1.26.6 +## +kubeTargetVersionOverride: "" + +## Allow kubeVersion to be overridden while creating the ingress +## +kubeVersionOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +## Install Prometheus Operator CRDs +## +crds: + enabled: true + +## custom Rules to override "for" and "severity" in defaultRules +## +customRules: {} + # AlertmanagerFailedReload: + # for: 3m + # AlertmanagerMembersInconsistent: + # for: 5m + # severity: "warning" + +## Create default rules for monitoring the cluster +## +defaultRules: + create: true + rules: + alertmanager: true + etcd: true + configReloaders: true + general: true + k8sContainerCpuUsageSecondsTotal: true + k8sContainerMemoryCache: true + k8sContainerMemoryRss: true + k8sContainerMemorySwap: true + k8sContainerResource: true + k8sContainerMemoryWorkingSetBytes: true + k8sPodOwner: true + kubeApiserverAvailability: true + kubeApiserverBurnrate: true + kubeApiserverHistogram: true + kubeApiserverSlos: true + kubeControllerManager: true + kubelet: true + kubeProxy: true + kubePrometheusGeneral: true + kubePrometheusNodeRecording: true + kubernetesApps: true + kubernetesResources: true + kubernetesStorage: true + kubernetesSystem: true + kubeSchedulerAlerting: true + kubeSchedulerRecording: true + kubeStateMetrics: true + network: true + node: true + nodeExporterAlerting: true + nodeExporterRecording: true + prometheus: true + prometheusOperator: true + windows: true + + ## Reduce app namespace alert scope + appNamespacesTarget: ".*" + + ## Set keep_firing_for for all alerts + keepFiringFor: "" + + ## Labels for default rules + labels: {} + ## Annotations for default rules + annotations: {} + + ## Additional labels for PrometheusRule alerts + additionalRuleLabels: {} + + ## Additional annotations for PrometheusRule alerts + additionalRuleAnnotations: {} + + ## Additional labels for specific PrometheusRule alert groups + additionalRuleGroupLabels: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + ## Additional annotations for specific PrometheusRule alerts groups + additionalRuleGroupAnnotations: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + additionalAggregationLabels: [] + + ## Prefix for runbook URLs. Use this to override the first part of the runbookURLs that is common to all rules. + runbookUrl: "https://runbooks.prometheus-operator.dev/runbooks" + + ## Disabled PrometheusRule alerts + disabled: {} + # KubeAPIDown: true + # NodeRAIDDegraded: true + +## Deprecated way to provide custom recording or alerting rules to be deployed into the cluster. +## +# additionalPrometheusRules: [] +# - name: my-rule-file +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## Provide custom recording or alerting rules to be deployed into the cluster. +## +additionalPrometheusRulesMap: {} +# rule-name: +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## +global: + cattle: + psp: + enabled: false + + systemDefaultRegistry: "" + ## Windows Monitoring + ## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-windows-exporter + ## + ## Deploys a DaemonSet of Prometheus exporters based on https://github.com/prometheus-community/windows_exporter. + ## Every Windows host must have a wins version of 0.1.0+ to use this chart (default as of Rancher 2.5.8). + ## To upgrade wins versions on Windows hosts, see https://github.com/rancher/wins/tree/master/charts/rancher-wins-upgrader. + ## + windows: + enabled: false + seLinux: + enabled: false + kubectl: + repository: rancher/kubectl + tag: v1.20.2 + pullPolicy: IfNotPresent + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Global image registry to use if it needs to be overriden for some specific use cases (e.g local registries, custom images, ...) + ## + imageRegistry: docker.io + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + # or + # - "image-pull-secret" + +windowsMonitoring: + ## Deploys the windows-exporter and Windows-specific dashboards and rules (job name must be 'windows-exporter') + enabled: false + +## Configuration for prometheus-windows-exporter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-windows-exporter +## +prometheus-windows-exporter: + ## Enable ServiceMonitor and set Kubernetes label to use as a job label + ## + prometheus: + monitor: + enabled: true + jobLabel: jobLabel + + releaseLabel: true + + ## Set job label to 'windows-exporter' as required by the default Prometheus rules and Grafana dashboards + ## + podLabels: + jobLabel: windows-exporter + + ## Enable memory and container metrics as required by the default Prometheus rules and Grafana dashboards + ## + config: |- + collectors: + enabled: '[defaults],memory,container' + +## Configuration for alertmanager +## ref: https://prometheus.io/docs/alerting/alertmanager/ +## +alertmanager: + + ## Deploy alertmanager + ## + enabled: true + + ## Annotations for Alertmanager + ## + annotations: {} + + ## Api that prometheus will use to communicate with alertmanager. Possible values are v1, v2 + ## + apiVersion: v2 + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + ## Configure pod disruption budgets for Alertmanager + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + ## Alertmanager configuration directives + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + config: + global: + resolve_timeout: 5m + inhibit_rules: + - source_matchers: + - 'severity = critical' + target_matchers: + - 'severity =~ warning|info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'severity = warning' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'alertname = InfoInhibitor' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - target_matchers: + - 'alertname = InfoInhibitor' + route: + group_by: ['namespace'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'null' + routes: + - receiver: 'null' + matchers: + - alertname = "Watchdog" + receivers: + - name: 'null' + templates: + - '/etc/alertmanager/config/*.tmpl' + + ## Alertmanager configuration directives (as string type, preferred over the config hash map) + ## stringConfig will be used only, if tplConfig is true + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + stringConfig: "" + + ## Pass the Alertmanager configuration directives through Helm's templating + ## engine. If the Alertmanager configuration contains Alertmanager templates, + ## they'll need to be properly escaped so that they are not interpreted by + ## Helm + ## ref: https://helm.sh/docs/developing_charts/#using-the-tpl-function + ## https://prometheus.io/docs/alerting/configuration/#tmpl_string + ## https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + tplConfig: false + + ## Alertmanager template files to format alerts + ## By default, templateFiles are placed in /etc/alertmanager/config/ and if + ## they have a .tmpl file suffix will be loaded. See config.templates above + ## to change, add other suffixes. If adding other suffixes, be sure to update + ## config.templates above to include those suffixes. + ## ref: https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + ## + + templateFiles: + rancher_defaults.tmpl: |- + {{- define "slack.rancher.text" -}} + {{ template "rancher.text_multiple" . }} + {{- end -}} + + {{- define "rancher.text_multiple" -}} + *[GROUP - Details]* + One or more alarms in this group have triggered a notification. + + {{- if gt (len .GroupLabels.Values) 0 }} + *Group Labels:* + {{- range .GroupLabels.SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- if .ExternalURL }} + *Link to AlertManager:* {{ .ExternalURL }} + {{- end }} + + {{- range .Alerts }} + {{ template "rancher.text_single" . }} + {{- end }} + {{- end -}} + + {{- define "rancher.text_single" -}} + {{- if .Labels.alertname }} + *[ALERT - {{ .Labels.alertname }}]* + {{- else }} + *[ALERT]* + {{- end }} + {{- if .Labels.severity }} + *Severity:* `{{ .Labels.severity }}` + {{- end }} + {{- if .Labels.cluster }} + *Cluster:* {{ .Labels.cluster }} + {{- end }} + {{- if .Annotations.summary }} + *Summary:* {{ .Annotations.summary }} + {{- end }} + {{- if .Annotations.message }} + *Message:* {{ .Annotations.message }} + {{- end }} + {{- if .Annotations.description }} + *Description:* {{ .Annotations.description }} + {{- end }} + {{- if .Annotations.runbook_url }} + *Runbook URL:* <{{ .Annotations.runbook_url }}|:spiral_note_pad:> + {{- end }} + {{- with .Labels }} + {{- with .Remove (stringSlice "alertname" "severity" "cluster") }} + {{- if gt (len .) 0 }} + *Additional Labels:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Annotations }} + {{- with .Remove (stringSlice "summary" "message" "description" "runbook_url") }} + {{- if gt (len .) 0 }} + *Additional Annotations:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end -}} + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + + labels: {} + + ## Override ingress to a different defined port on the service + # servicePort: 8081 + ## Override ingress to a different service then the default, this is useful if you need to + ## point to a specific instance of the alertmanager (eg kube-prometheus-stack-alertmanager-0) + # serviceName: kube-prometheus-stack-alertmanager-0 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - alertmanager.domain.com + + ## Paths to use for ingress rules - one path should match the alertmanagerSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Alertmanager Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: alertmanager-general-tls + # hosts: + # - alertmanager.example.com + + ## Configuration for Alertmanager secret + ## + secret: + annotations: {} + + # by default the alertmanager secret is not overwritten if it already exists + recreateIfExists: false + + ## Configuration for creating an Ingress that will map to each Alertmanager replica service + ## alertmanager.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for alertmanager per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "alertmanager" + + ## Configuration for Alertmanager service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Alertmanager Service to listen on + ## + port: 9093 + ## To be used with a proxy extraContainer port + ## + targetPort: 9093 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30903 + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + + ## Additional ports to open for Alertmanager service + ## + additionalPorts: [] + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a separate Service for each statefulset Alertmanager replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Alertmanager Service per replica to listen on + ## + port: 9093 + + ## To be used with a proxy extraContainer port + targetPort: 9093 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30904 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a ServiceMonitor for AlertManager + ## + serviceMonitor: + ## If true, a ServiceMonitor will be created for the AlertManager service. + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## enableHttp2: Whether to enable HTTP2. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint + enableHttp2: true + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting alertmanagerSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec + ## + alertmanagerSpec: + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the Alertmanager pods. + ## + podMetadata: {} + + ## Image of Alertmanager + ## + image: + repository: rancher/mirrored-prometheus-alertmanager + tag: v0.27.0 + sha: "" + + ## If true then the user will be responsible to provide a secret with alertmanager configuration + ## So when true the config part will be ignored (including templateFiles) and the one in the secret will be used + ## + useExistingSecret: false + + ## Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the + ## Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/. + ## + secrets: [] + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. + ## The ConfigMaps are mounted into /etc/alertmanager/configmaps/. + ## + configMaps: [] + + ## ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for + ## this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config. + ## + # configSecret: + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerwebspec + web: {} + + ## AlertmanagerConfigs to be selected to merge and configure Alertmanager with. + ## + alertmanagerConfigSelector: {} + ## Example which selects all alertmanagerConfig resources + ## with label "alertconfig" with values any of "example-config" or "example-config-2" + # alertmanagerConfigSelector: + # matchExpressions: + # - key: alertconfig + # operator: In + # values: + # - example-config + # - example-config-2 + # + ## Example which selects all alertmanagerConfig resources with label "role" set to "example-config" + # alertmanagerConfigSelector: + # matchLabels: + # role: example-config + + ## Namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace. + ## + alertmanagerConfigNamespaceSelector: {} + ## Example which selects all namespaces + ## with label "alertmanagerconfig" with values any of "example-namespace" or "example-namespace-2" + # alertmanagerConfigNamespaceSelector: + # matchExpressions: + # - key: alertmanagerconfig + # operator: In + # values: + # - example-namespace + # - example-namespace-2 + + ## Example which selects all namespaces with label "alertmanagerconfig" set to "enabled" + # alertmanagerConfigNamespaceSelector: + # matchLabels: + # alertmanagerconfig: enabled + + ## AlermanagerConfig to be used as top level configuration + ## + alertmanagerConfiguration: {} + ## Example with select a global alertmanagerconfig + # alertmanagerConfiguration: + # name: global-alertmanager-Configuration + + ## Defines the strategy used by AlertmanagerConfig objects to match alerts. eg: + ## + alertmanagerConfigMatcherStrategy: {} + ## Example with use OnNamespace strategy + # alertmanagerConfigMatcherStrategy: + # type: OnNamespace + + ## Define Log Format + # Use logfmt (default) or json logging + logFormat: logfmt + + ## Log level for Alertmanager to be configured with. + ## + logLevel: info + + ## Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the + ## running cluster equal to the expected size. + replicas: 1 + + ## Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression + ## [0-9]+(ms|s|m|h) (milliseconds seconds minutes hours). + ## + retention: 120h + + ## Storage is the definition of how storage will be used by the Alertmanager instances. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storage: {} + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + + ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false + ## + externalUrl: + + ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, + ## but the server serves requests under a different route prefix. For example for use with kubectl proxy. + ## + routePrefix: / + + ## scheme: HTTP scheme to use. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when connect to the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + ## If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. + ## + paused: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Define resources requests and limits for single Pods. + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: + memory: 500Mi + cpu: 1000m + requests: + memory: 100Mi + cpu: 100m + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + ## + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the alertmanager instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## If specified, the pod's tolerations. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: alertmanager + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. + ## Note this is only for the Alertmanager UI, not the gossip communication. + ## + listenLocal: false + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. + ## + containers: [] + # containers: + # - name: oauth-proxy + # image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1 + # args: + # - --upstream=http://127.0.0.1:9093 + # - --http-address=0.0.0.0:8081 + # - --metrics-address=0.0.0.0:8082 + # - ... + # ports: + # - containerPort: 8081 + # name: oauth-proxy + # protocol: TCP + # - containerPort: 8082 + # name: oauth-metrics + # protocol: TCP + # resources: {} + + # Additional volumes on the output StatefulSet definition. + volumes: [] + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. + ## + additionalPeers: [] + + ## PortName to use for Alert Manager. + ## + portName: "http-web" + + ## ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 + ## + clusterAdvertiseAddress: false + + ## clusterGossipInterval determines interval between gossip attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterGossipInterval: "" + + ## clusterPeerTimeout determines timeout for cluster peering. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPeerTimeout: "" + + ## clusterPushpullInterval determines interval between pushpull attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPushpullInterval: "" + + ## ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. + ## Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. + forceEnableClusterMode: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use alertmanager.alertmanagerSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + +## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +## +grafana: + enabled: true + namespaceOverride: "" + + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + grafana.ini: + users: + auto_assign_org_role: Viewer + auth: + disable_login_form: false + auth.anonymous: + enabled: true + org_role: Viewer + auth.basic: + enabled: false + dashboards: + # Modify this value to change the default dashboard shown on the main Grafana page + default_home_dashboard_path: /tmp/dashboards/rancher-default-home.json + security: + # Required to embed dashboards in Rancher Cluster Overview Dashboard on Cluster Explorer + allow_embedding: true + + deploymentStrategy: + type: Recreate + + ## ForceDeployDatasources Create datasource configmap even if grafana deployment has been disabled + ## + forceDeployDatasources: false + + ## ForceDeployDashboard Create dashboard configmap even if grafana deployment has been disabled + ## + forceDeployDashboards: false + + ## Deploy default dashboards + ## + defaultDashboardsEnabled: true + + # Additional options for defaultDashboards + defaultDashboards: + # The default namespace to place defaultDashboards within + namespace: cattle-dashboards + # Whether to create the default namespace as a Helm managed namespace or use an existing namespace + # If false, the defaultDashboards.namespace will be created as a Helm managed namespace + useExistingNamespace: false + # Whether the Helm managed namespace created by this chart should be left behind on a Helm uninstall + # If you place other dashboards in this namespace, then they will be deleted on a helm uninstall + # Ignore if useExistingNamespace is true + cleanupOnUninstall: false + + ## Timezone for the default dashboards + ## Other options are: browser or a specific timezone, i.e. Europe/Luxembourg + ## + defaultDashboardsTimezone: utc + + ## Editable flag for the default dashboards + ## + defaultDashboardsEditable: true + + adminPassword: prom-operator + + ingress: + ## If true, Grafana Ingress will be created + ## + enabled: false + + ## IngressClassName for Grafana Ingress. + ## Should be provided if Ingress is enable. + ## + # ingressClassName: nginx + + ## Annotations for Grafana Ingress + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + + ## Labels to be added to the Ingress + ## + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enable. + ## + # hosts: + # - grafana.domain.com + hosts: [] + + ## Path for grafana ingress + path: / + + ## TLS configuration for grafana Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: grafana-general-tls + # hosts: + # - grafana.example.com + + # # To make Grafana persistent (Using Statefulset) + # # + # persistence: + # enabled: true + # type: sts + # storageClassName: "storageClassName" + # accessModes: + # - ReadWriteOnce + # size: 20Gi + # finalizers: + # - kubernetes.io/pvc-protection + + serviceAccount: + create: true + autoMount: true + + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + searchNamespace: cattle-dashboards + labelValue: "1" + + # Support for new table panels, when enabled grafana auto migrates the old table panels to newer table panels + enableNewTablePanelSyntax: false + + ## Annotations for Grafana dashboard configmaps + ## + annotations: {} + multicluster: + global: + enabled: false + etcd: + enabled: false + provider: + allowUiUpdates: false + datasources: + enabled: true + defaultDatasourceEnabled: true + isDefaultDatasource: true + + uid: prometheus + + ## URL of prometheus datasource + ## + # url: http://prometheus-stack-prometheus:9090/ + + ## Prometheus request timeout in seconds + # timeout: 30 + + # If not defined, will use prometheus.prometheusSpec.scrapeInterval or its default + # defaultDatasourceScrapeInterval: 15s + + ## Annotations for Grafana datasource configmaps + ## + annotations: {} + + ## Set method for HTTP to send query to datasource + httpMethod: POST + + ## Create datasource for each Pod of Prometheus StatefulSet; + ## this uses headless service `prometheus-operated` which is + ## created by Prometheus Operator + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/0fee93e12dc7c2ea1218f19ae25ec6b893460590/pkg/prometheus/statefulset.go#L255-L286 + createPrometheusReplicasDatasources: false + label: grafana_datasource + labelValue: "1" + + ## Field with internal link pointing to existing data source in Grafana. + ## Can be provisioned via additionalDataSources + exemplarTraceIdDestinations: {} + # datasourceUid: Jaeger + # traceIdLabelName: trace_id + alertmanager: + enabled: true + uid: alertmanager + handleGrafanaManagedAlerts: false + implementation: prometheus + + extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # configMap: certs-configmap + # readOnly: true + + deleteDatasources: [] + # - name: example-datasource + # orgId: 1 + + ## Configure additional grafana datasources (passed through tpl) + ## ref: http://docs.grafana.org/administration/provisioning/#datasources + additionalDataSources: [] + # - name: prometheus-sample + # access: proxy + # basicAuth: true + # basicAuthPassword: pass + # basicAuthUser: daco + # editable: false + # jsonData: + # tlsSkipVerify: true + # orgId: 1 + # type: prometheus + # url: https://{{ printf "%s-prometheus.svc" .Release.Name }}:9090 + # version: 1 + + ## Passed to grafana subchart and used by servicemonitor below + ## + service: + portName: nginx-http + ## Port for Grafana Service to listen on + ## + port: 80 + ## To be used with a proxy extraContainer port + ## + targetPort: 8080 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30950 + ## Service type + ## + type: ClusterIP + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod + extraContainers: | + - name: grafana-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }}" + ports: + - containerPort: 8080 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: grafana-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## Volumes that can be used in containers + extraContainerVolumes: + - name: nginx-home + emptyDir: {} + - name: grafana-nginx + configMap: + name: grafana-nginx-proxy-config + items: + - key: nginx.conf + mode: 438 + path: nginx.conf + + ## If true, create a serviceMonitor for grafana + ## + serviceMonitor: + # If true, a ServiceMonitor CRD is created for a prometheus operator + # https://github.com/coreos/prometheus-operator + # + enabled: true + + # Path to use for scraping metrics. Might be different if server.root_url is set + # in grafana.ini + path: "/metrics" + + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + + # labels for the ServiceMonitor + labels: {} + + # Scrape interval. If not set, the Prometheus default scrape interval is used. + # + interval: "" + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + resources: + limits: + memory: 200Mi + cpu: 200m + requests: + memory: 100Mi + cpu: 100m + + testFramework: + enabled: false + +## Flag to disable all the kubernetes component scrapers +## +kubernetesServiceMonitors: + enabled: true + +## Component scraping the kube api server +## +kubeApiServer: + enabled: true + tlsConfig: + serverName: kubernetes + insecureSkipVerify: false + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: component + selector: + matchLabels: + component: apiserver + provider: kubernetes + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: + # Drop excessively noisy apiserver buckets. + - action: drop + regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50) + sourceLabels: + - __name__ + - le + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: + # - __meta_kubernetes_namespace + # - __meta_kubernetes_service_name + # - __meta_kubernetes_endpoint_port_name + # action: keep + # regex: default;kubernetes;https + # - targetLabel: __address__ + # replacement: kubernetes.default.svc:443 + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kubelet and kubelet-hosted cAdvisor +## +kubelet: + enabled: true + namespace: kube-system + + serviceMonitor: + ## Attach metadata to discovered targets. Requires Prometheus v2.45 for endpoints created by the operator. + ## + attachMetadata: + node: false + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## If true, Prometheus use (respect) labels provided by exporter. + ## + honorLabels: true + + ## If true, Prometheus ingests metrics with timestamp provided by exporter. If false, Prometheus ingests metrics with timestamp of scrape. + ## + honorTimestamps: true + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Enable scraping the kubelet over https. For requirements to enable this see + ## https://github.com/prometheus-operator/prometheus-operator/issues/926 + ## + https: true + + ## Enable scraping /metrics/cadvisor from kubelet's service + ## + cAdvisor: true + + ## Enable scraping /metrics/probes from kubelet's service + ## + probes: true + + ## Enable scraping /metrics/resource from kubelet's service + ## This is disabled by default because container metrics are already exposed by cAdvisor + ## + resource: false + # From kubernetes 1.18, /metrics/resource/v1alpha1 renamed to /metrics/resource + resourcePath: "/metrics/resource/v1alpha1" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + cAdvisorMetricRelabelings: + # Drop less useful container CPU metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_cpu_(cfs_throttled_seconds_total|load_average_10s|system_seconds_total|user_seconds_total)' + # Drop less useful container / always zero filesystem metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_fs_(io_current|io_time_seconds_total|io_time_weighted_seconds_total|reads_merged_total|sector_reads_total|sector_writes_total|writes_merged_total)' + # Drop less useful / always zero container memory metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_memory_(mapped_file|swap)' + # Drop less useful container process metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_(file_descriptors|tasks_state|threads_max)' + # Drop container spec metrics that overlap with kube-state-metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_spec.*' + # Drop cgroup metrics with no pod. + - sourceLabels: [id, pod] + action: drop + regex: '.+;' + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesMetricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + cAdvisorRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + resourceRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + relabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kube controller manager +## +kubeControllerManager: + enabled: false + + ## If your kube controller manager is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeControllerManager.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.22. + ## + port: null + targetPort: null + # selector: + # component: kube-controller-manager + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-controller-manager + + ## Enable scraping kube-controller-manager over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + # Skip TLS certificate validation when scraping + insecureSkipVerify: null + + # Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping coreDns. Use either this or kubeDns +## +coreDns: + enabled: true + service: + enabled: true + port: 9153 + targetPort: 9153 + # selector: + # k8s-app: kube-dns + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kubeDns. Use either this or coreDns +## +kubeDns: + enabled: false + service: + dnsmasq: + port: 10054 + targetPort: 10054 + skydns: + port: 10055 + targetPort: 10055 + # selector: + # k8s-app: kube-dns + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqMetricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqRelabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping etcd +## +kubeEtcd: + enabled: false + + ## If your etcd is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## Etcd service. If using kubeEtcd.endpoints only the port and targetPort are used + ## + service: + enabled: true + port: 2381 + targetPort: 2381 + # selector: + # component: etcd + + ## Configure secure access to the etcd cluster by loading a secret into prometheus and + ## specifying security configuration below. For example, with a secret named etcd-client-cert + ## + ## serviceMonitor: + ## scheme: https + ## insecureSkipVerify: false + ## serverName: localhost + ## caFile: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + ## certFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client + ## keyFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + ## + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + scheme: http + insecureSkipVerify: false + serverName: "" + caFile: "" + certFile: "" + keyFile: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: etcd + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube scheduler +## +kubeScheduler: + enabled: false + + ## If your kube scheduler is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeScheduler.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.23. + ## + port: null + targetPort: null + # selector: + # component: kube-scheduler + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + ## Enable scraping kube-scheduler over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-scheduler + + ## Skip TLS certificate validation when scraping + insecureSkipVerify: null + + ## Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube proxy +## +kubeProxy: + enabled: false + + ## If your kube proxy is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + service: + enabled: true + port: 10249 + targetPort: 10249 + # selector: + # k8s-app: kube-proxy + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-proxy + + ## Enable scraping kube-proxy over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## + https: false + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube state metrics +## +kubeStateMetrics: + enabled: true + +## Configuration for kube-state-metrics subchart +## +kube-state-metrics: + namespaceOverride: "" + rbac: + create: true + releaseLabel: true + prometheus: + monitor: + enabled: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape Timeout. If not set, the Prometheus default scrape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + # Keep labels from scraped data, overriding server-side labels + ## + honorLabels: true + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + selfMonitor: + enabled: false + +## Deploy node exporter as a daemonset to all nodes +## +nodeExporter: + enabled: true + operatingSystems: + linux: + enabled: true + darwin: + enabled: true + + ## ForceDeployDashboard Create dashboard configmap even if nodeExporter deployment has been disabled + ## + forceDeployDashboards: false + +## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + namespaceOverride: "" + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + releaseLabel: true + extraArgs: + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ + service: + portName: http-metrics + prometheus: + monitor: + enabled: true + + jobLabel: jobLabel + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__] + # separator: ; + # regex: ^node_mountstats_nfs_(event|operations|transport)_.+ + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Manages Prometheus and Alertmanager components +## +prometheusOperator: + enabled: true + + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-operator' by default + fullnameOverride: "" + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Strategy of the deployment + ## + strategy: {} + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # Users who are deploying this chart in GKE private clusters will need to add firewall rules to expose this port for admissions webhooks + internalPort: 8443 + + ## Admission webhook support for PrometheusRules resources added in Prometheus Operator 0.30 can be enabled to prevent incorrectly formatted + ## rules from making their way into prometheus and potentially preventing the container from starting + admissionWebhooks: + ## Valid values: Fail, Ignore, IgnoreOnInstallOnly + ## IgnoreOnInstallOnly - If Release.IsInstall returns "true", set "Ignore" otherwise "Fail" + failurePolicy: "" + ## The default timeoutSeconds is 10 and the maximum value is 30. + timeoutSeconds: 10 + enabled: true + ## A PEM encoded CA bundle which will be used to validate the webhook's server certificate. + ## If unspecified, system trust roots on the apiserver are used. + caBundle: "" + ## If enabled, generate a self-signed certificate, then patch the webhook configurations with the generated data. + ## On chart upgrades (or if the secret exists) the cert will not be re-generated. You can use this to provide your own + ## certs ahead of time if you wish. + ## + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + + namespaceSelector: {} + + deployment: + enabled: false + + ## Number of replicas + ## + replicas: 1 + + ## Strategy of the deployment + ## + strategy: {} + + # Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: {} + # maxUnavailable: 1 + # minAvailable: 1 + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # The default webhook port is 10250 in order to work out-of-the-box in GKE private clusters and avoid adding firewall rules. + internalPort: 10250 + + ## Service account for Prometheus Operator Webhook to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + automountServiceAccountToken: false + create: true + name: "" + + ## Configuration for Prometheus operator Webhook service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 31080 + + nodePortTls: 31443 + + ## Additional ports to open for Prometheus operator Webhook service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator webhook deployment + # ## + labels: {} + + ## Annotations to add to the operator webhook deployment + ## + annotations: {} + + ## Labels to add to the operator webhook pod + ## + podLabels: {} + + ## Annotations to add to the operator webhook pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + ## Prometheus-operator webhook image + ## + image: + registry: quay.io + repository: rancher/mirrored-prometheus-operator-admission-webhook + # if not set appVersion field from Chart.yaml is used + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + + ## Liveness probe + ## + livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Readiness probe + ## + readinessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Resource limits & requests + ## + resources: {} + # limits: + # cpu: 200m + # memory: 200Mi + # requests: + # cpu: 100m + # memory: 100Mi + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + patch: + enabled: true + image: + repository: rancher/mirrored-ingress-nginx-kube-webhook-certgen + tag: v1.4.3 + sha: "" + pullPolicy: IfNotPresent + resources: {} + ## Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + podAnnotations: {} + nodeSelector: {} + affinity: {} + tolerations: [] + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 2000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + seccompProfile: + type: RuntimeDefault + + # Security context for create job container + createSecretJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Security context for patch job container + patchWebhookJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Use certmanager to generate webhook certs + certManager: + enabled: false + # self-signed root certificate + rootCert: + duration: "" # default to be 5y + admissionCert: + duration: "" # default to be 1y + # issuerRef: + # name: "issuer" + # kind: "ClusterIssuer" + + ## Namespaces to scope the interaction of the Prometheus Operator and the apiserver (allow list). + ## This is mutually exclusive with denyNamespaces. Setting this to an empty object will disable the configuration + ## + namespaces: {} + # releaseNamespace: true + # additional: + # - kube-system + + ## Namespaces not to scope the interaction of the Prometheus Operator (deny list). + ## + denyNamespaces: [] + + ## Filter namespaces to look for prometheus-operator custom resources + ## + alertmanagerInstanceNamespaces: [] + alertmanagerConfigNamespaces: [] + prometheusInstanceNamespaces: [] + thanosRulerInstanceNamespaces: [] + + ## The clusterDomain value will be added to the cluster.peer option of the alertmanager. + ## Without this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated:9094 (default value) + ## With this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated.namespace.svc.cluster-domain:9094 + ## + # clusterDomain: "cluster.local" + + networkPolicy: + ## Enable creation of NetworkPolicy resources. + ## + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # egress: + + ## match labels used in selector + # matchLabels: {} + + ## Service account for Prometheus Operator to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + + ## Configuration for Prometheus operator service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30080 + + nodePortTls: 30443 + + ## Additional ports to open for Prometheus operator service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator deployment + # ## + labels: {} + + ## Annotations to add to the operator deployment + ## + annotations: {} + + ## Labels to add to the operator pod + ## + podLabels: {} + + ## Annotations to add to the operator pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + kubeletService: + ## If true, the operator will create and maintain a service for scraping kubelets + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/helm/prometheus-operator/README.md + ## + enabled: true + namespace: kube-system + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-kubelet' by default + name: "" + + ## Create a servicemonitor for the operator + ## + serviceMonitor: + ## If true, create a serviceMonitor for prometheus operator + ## + selfMonitor: true + + ## Labels for ServiceMonitor + additionalLabels: {} + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape timeout. If not set, the Prometheus default scrape timeout is used. + scrapeTimeout: "" + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Resource limits & requests + ## + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + + ## Operator Environment + ## env: + ## VARIABLE: value + env: + GOGC: "30" + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Enable vertical pod autoscaler support for prometheus-operator + verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + updateMode: Auto + + ## Prometheus-operator image + ## + image: + repository: rancher/mirrored-prometheus-operator-prometheus-operator + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Prometheus image to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImage: prometheus/prometheus + + ## Prometheus image registry to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImageRegistry: quay.io + + ## Alertmanager image to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImage: prometheus/alertmanager + + ## Alertmanager image registry to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImageRegistry: quay.io + + ## Prometheus-config-reloader + ## + prometheusConfigReloader: + image: + repository: rancher/mirrored-prometheus-operator-prometheus-config-reloader + tag: v0.72.0 + sha: "" + + # add prometheus config reloader liveness and readiness probe. Default: false + enableProbe: false + + # resource config for prometheusConfigReloader + resources: {} + # requests: + # cpu: 200m + # memory: 50Mi + # limits: + # cpu: 200m + # memory: 50Mi + + ## Thanos side-car image when configured + ## + thanosImage: + repository: rancher/mirrored-thanos-thanos + tag: v0.34.1 + sha: "" + + ## Set a Label Selector to filter watched prometheus and prometheusAgent + ## + prometheusInstanceSelector: "" + + ## Set a Label Selector to filter watched alertmanager + ## + alertmanagerInstanceSelector: "" + + ## Set a Label Selector to filter watched thanosRuler + thanosRulerInstanceSelector: "" + + ## Set a Field Selector to filter watched secrets + ## + secretFieldSelector: "type!=kubernetes.io/dockercfg,type!=kubernetes.io/service-account-token,type!=helm.sh/release.v1" + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## Additional volumes + ## + extraVolumes: [] + + ## Additional volume mounts + ## + extraVolumeMounts: [] + +## Deploy a Prometheus instance +## +prometheus: + enabled: true + + ## Toggle prometheus into agent mode + ## Note many of features described below (e.g. rules, query, alerting, remote read, thanos) will not work in agent mode. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/designs/prometheus-agent.md + ## + agentMode: false + + ## Annotations for Prometheus + ## + annotations: {} + + ## Configure network policy for the prometheus + networkPolicy: + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # endpointSelector: + # egress: + # ingress: + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app: prometheus + + ## Service account for Prometheuses to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + # Service for thanos service discovery on sidecar + # Enable this can make Thanos Query can use + # `--store=dnssrv+_grpc._tcp.${kube-prometheus-stack.fullname}-thanos-discovery.${namespace}.svc.cluster.local` to discovery + # Thanos sidecar on prometheus nodes + # (Please remember to change ${kube-prometheus-stack.fullname} and ${namespace}. Not just copy and paste!) + thanosService: + enabled: false + annotations: {} + labels: {} + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## ClusterIP to assign + # Default is to make this a headless service ("None") + clusterIP: "None" + + ## Port to expose on each node, if service type is NodePort + ## + nodePort: 30901 + httpNodePort: 30902 + + # ServiceMonitor to scrape Sidecar metrics + # Needs thanosService to be enabled as well + thanosServiceMonitor: + enabled: false + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + metricRelabelings: [] + + ## relabel configs to apply to samples before ingestion. + relabelings: [] + + # Service for external access to sidecar + # Enabling this creates a service to expose thanos-sidecar outside the cluster. + thanosServiceExternal: + enabled: false + annotations: {} + labels: {} + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: LoadBalancer + + ## Port to expose on each node + ## + nodePort: 30901 + httpNodePort: 30902 + + ## Configuration for Prometheus service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Prometheus Service to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 8081 + + ## Port for Prometheus Reloader to listen on + ## + reloaderWebPort: 8080 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30090 + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Additional ports to open for Prometheus service + ## + additionalPorts: [] + # additionalPorts: + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + ## Consider that all endpoints are considered "ready" even if the Pods themselves are not + ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec + publishNotReadyAddresses: false + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Configuration for creating a separate Service for each statefulset Prometheus replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Prometheus Service per replica to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30091 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configure pod disruption budgets for Prometheus + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + # Ingress exposes thanos sidecar outside the cluster + thanosIngress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + servicePort: 10901 + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30901 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - thanos-gateway.domain.com + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Thanos Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: thanos-gateway-tls + # hosts: + # - thanos-gateway.domain.com + # + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Redirect ingress to an additional defined port on the service + # servicePort: 8081 + + ## Hostnames. + ## Must be provided if Ingress is enabled. + ## + # hosts: + # - prometheus.domain.com + hosts: [] + + ## Paths to use for ingress rules - one path should match the prometheusSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Prometheus Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: prometheus-general-tls + # hosts: + # - prometheus.example.com + + ## Configuration for creating an Ingress that will map to each Prometheus replica service + ## prometheus.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for Prometheus per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "prometheus" + + ## Configure additional options for default pod security policy for Prometheus + ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + allowedCapabilities: [] + allowedHostPaths: [] + volumes: [] + + serviceMonitor: + ## If true, create a serviceMonitor for prometheus + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting prometheusSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec + ## + prometheusSpec: + ## If true, pass --storage.tsdb.max-block-duration=2h to prometheus. This is already done if using Thanos + ## + disableCompaction: false + ## APIServerConfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#apiserverconfig + ## + apiserverConfig: {} + + ## Allows setting additional arguments for the Prometheus container + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.Prometheus + additionalArgs: [] + + ## Interval between consecutive scrapes. + ## Defaults to 30s. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/release-0.44/pkg/prometheus/promcfg.go#L180-L183 + ## + scrapeInterval: "30s" + + ## Number of seconds to wait for target to respond before erroring + ## + # scrapeTimeout: "30s" + + ## Interval between consecutive evaluations. + ## + evaluationInterval: "30s" + + ## ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. + ## + listenLocal: false + + ## EnableAdminAPI enables Prometheus the administrative HTTP API which includes functionality such as deleting time series. + ## This is disabled by default. + ## ref: https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis + ## + enableAdminAPI: false + + ## Sets version of Prometheus overriding the Prometheus version as derived + ## from the image tag. Useful in cases where the tag does not follow semver v2. + version: "" + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#webtlsconfig + web: {} + + ## Exemplars related settings that are runtime reloadable. + ## It requires to enable the exemplar storage feature to be effective. + exemplars: "" + ## Maximum number of exemplars stored in memory for all series. + ## If not set, Prometheus uses its default value. + ## A value of zero or less than zero disables the storage. + # maxSize: 100000 + + # EnableFeatures API enables access to Prometheus disabled features. + # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/ + enableFeatures: [] + # - exemplar-storage + + ## Image of Prometheus. + ## + image: + repository: rancher/mirrored-prometheus-prometheus + tag: v2.50.1 + sha: "" + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: prometheus + + ## Alertmanagers to which alerts will be sent + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerendpoints + ## + ## Default configuration will connect to the alertmanager deployed as part of this release + ## + alertingEndpoints: [] + # - name: "" + # namespace: "" + # port: http + # scheme: http + # pathPrefix: "" + # tlsConfig: {} + # bearerTokenFile: "" + # apiVersion: v2 + + ## External labels to add to any time series or alerts when communicating with external systems + ## + externalLabels: {} + + ## enable --web.enable-remote-write-receiver flag on prometheus-server + ## + enableRemoteWriteReceiver: false + + ## Name of the external label used to denote replica name + ## + replicaExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote replica name + ## + replicaExternalLabelNameClear: false + + ## Name of the external label used to denote Prometheus instance name + ## + prometheusExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote Prometheus instance name + ## + prometheusExternalLabelNameClear: false + + ## External URL at which Prometheus will be reachable. + ## + externalUrl: "" + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not + ## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated + ## with the new list of secrets. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The ConfigMaps are mounted into /etc/prometheus/configmaps/. + ## + configMaps: [] + + ## QuerySpec defines the query command line flags when starting Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#queryspec + ## + query: {} + + ## If nil, select own namespace. Namespaces to be selected for PrometheusRules discovery. + ruleNamespaceSelector: {} + ## Example which selects PrometheusRules in namespaces with label "prometheus" set to "somelabel" + # ruleNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.ruleSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the PrometheusRule resources created + ## + ruleSelectorNilUsesHelmValues: false + + ## PrometheusRules to be selected for target discovery. + ## If {}, select all PrometheusRules + ## + ruleSelector: {} + ## Example which select all PrometheusRules resources + ## with label "prometheus" with values any of "example-rules" or "example-rules-2" + # ruleSelector: + # matchExpressions: + # - key: prometheus + # operator: In + # values: + # - example-rules + # - example-rules-2 + # + ## Example which select all PrometheusRules resources with label "role" set to "example-rules" + # ruleSelector: + # matchLabels: + # role: example-rules + + ## If true, a nil or {} value for prometheus.prometheusSpec.serviceMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the servicemonitors created + ## + serviceMonitorSelectorNilUsesHelmValues: false + + ## ServiceMonitors to be selected for target discovery. + ## If {}, select all ServiceMonitors + ## + serviceMonitorSelector: {} + ## Example which selects ServiceMonitors with label "prometheus" set to "somelabel" + # serviceMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for ServiceMonitor discovery. + ## + serviceMonitorNamespaceSelector: {} + ## Example which selects ServiceMonitors in namespaces with label "prometheus" set to "somelabel" + # serviceMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.podMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the podmonitors created + ## + podMonitorSelectorNilUsesHelmValues: false + + ## PodMonitors to be selected for target discovery. + ## If {}, select all PodMonitors + ## + podMonitorSelector: {} + ## Example which selects PodMonitors with label "prometheus" set to "somelabel" + # podMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for PodMonitor discovery. + podMonitorNamespaceSelector: {} + ## Example which selects PodMonitor in namespaces with label "prometheus" set to "somelabel" + # podMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.probeSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the probes created + ## + probeSelectorNilUsesHelmValues: true + + ## Probes to be selected for target discovery. + ## If {}, select all Probes + ## + probeSelector: {} + ## Example which selects Probes with label "prometheus" set to "somelabel" + # probeSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for Probe discovery. + probeNamespaceSelector: {} + ## Example which selects Probe in namespaces with label "prometheus" set to "somelabel" + # probeNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.scrapeConfigSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the scrapeConfigs created + ## + scrapeConfigSelectorNilUsesHelmValues: true + + ## scrapeConfigs to be selected for target discovery. + ## If {}, select all scrapeConfigs + ## + scrapeConfigSelector: {} + ## Example which selects scrapeConfigs with label "prometheus" set to "somelabel" + # scrapeConfigSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for scrapeConfig discovery. + scrapeConfigNamespaceSelector: {} + ## Example which selects scrapeConfig in namespaces with label "prometheus" set to "somelabel" + # scrapeConfigNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## How long to retain metrics + ## + retention: 10d + + ## Maximum size of metrics + ## + retentionSize: "" + + ## Allow out-of-order/out-of-bounds samples ingested into Prometheus for a specified duration + ## See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb + tsdb: + outOfOrderTimeWindow: 0s + + ## Enable compression of the write-ahead log using Snappy. + ## + walCompression: true + + ## If true, the Operator won't process any Prometheus configuration changes + ## + paused: false + + ## Number of replicas of each shard to deploy for a Prometheus deployment. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## + replicas: 1 + + ## EXPERIMENTAL: Number of shards to distribute targets onto. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. + ## Increasing shards will not reshard data either but it will continue to be available from the same instances. + ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. + ## Sharding is done on the content of the `__address__` target meta-label. + ## + shards: 1 + + ## Log level for Prometheus be configured in + ## + logLevel: info + + ## Log format for Prometheus be configured in + ## + logFormat: logfmt + + ## Prefix used to register routes, overriding externalUrl route. + ## Useful for proxies that rewrite URLs. + ## + routePrefix: / + + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the prometheus pods. + ## + podMetadata: {} + # labels: + # app: prometheus + # k8s-app: prometheus + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the prometheus instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## The remote_read spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotereadspec + remoteRead: [] + # - url: http://remote1/read + ## additionalRemoteRead is appended to remoteRead + additionalRemoteRead: [] + + ## The remote_write spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotewritespec + remoteWrite: [] + # - url: http://remote1/push + ## additionalRemoteWrite is appended to remoteWrite + additionalRemoteWrite: [] + + ## Enable/Disable Grafana dashboards provisioning for prometheus remote write feature + remoteWriteDashboards: false + + ## Resource limits & requests + ## + resources: + limits: + memory: 3000Mi + cpu: 1000m + requests: + memory: 750Mi + cpu: 750m + + ## Prometheus StorageSpec for persistent data + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storageSpec: {} + ## Using PersistentVolumeClaim + ## + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + ## Using tmpfs volume + ## + # emptyDir: + # medium: Memory + + # Additional volumes on the output StatefulSet definition. + volumes: + - name: nginx-home + emptyDir: {} + - name: prometheus-nginx + configMap: + name: prometheus-nginx-proxy-config + defaultMode: 438 + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations + ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form + ## as specified in the official Prometheus documentation: + ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are + ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility + ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible + ## scrape configs are going to break Prometheus after the upgrade. + ## AdditionalScrapeConfigs can be defined as a list or as a templated string. + ## + ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the + ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes + ## + additionalScrapeConfigs: [] + # - job_name: kube-etcd + # kubernetes_sd_configs: + # - role: node + # scheme: https + # tls_config: + # ca_file: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + # cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client + # key_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + # relabel_configs: + # - action: labelmap + # regex: __meta_kubernetes_node_label_(.+) + # - source_labels: [__address__] + # action: replace + # targetLabel: __address__ + # regex: ([^:;]+):(\d+) + # replacement: ${1}:2379 + # - source_labels: [__meta_kubernetes_node_name] + # action: keep + # regex: .*mst.* + # - source_labels: [__meta_kubernetes_node_name] + # action: replace + # targetLabel: node + # regex: (.*) + # replacement: ${1} + # metric_relabel_configs: + # - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone) + # action: labeldrop + # + ## If scrape config contains a repetitive section, you may want to use a template. + ## In the following example, you can see how to define `gce_sd_configs` for multiple zones + # additionalScrapeConfigs: | + # - job_name: "node-exporter" + # gce_sd_configs: + # {{range $zone := .Values.gcp_zones}} + # - project: "project1" + # zone: "{{$zone}}" + # port: 9100 + # {{end}} + # relabel_configs: + # ... + + + ## If additional scrape configurations are already deployed in a single secret file you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalScrapeConfigs + additionalScrapeConfigsSecret: {} + # enabled: false + # name: + # key: + + ## additionalPrometheusSecretsAnnotations allows to add annotations to the kubernetes secret. This can be useful + ## when deploying via spinnaker to disable versioning on the secret, strategy.spinnaker.io/versioned: 'false' + additionalPrometheusSecretsAnnotations: {} + + ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified + ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#. + ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. + ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this + ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release + ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. + ## + additionalAlertManagerConfigs: [] + # - consul_sd_configs: + # - server: consul.dev.test:8500 + # scheme: http + # datacenter: dev + # tag_separator: ',' + # services: + # - metrics-prometheus-alertmanager + + ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertManagerConfigs + additionalAlertManagerConfigsSecret: {} + # name: + # key: + # optional: false + + ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended + ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the + ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. + ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the + ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel + ## configs are going to break Prometheus after the upgrade. + ## + additionalAlertRelabelConfigs: [] + # - separator: ; + # regex: prometheus_replica + # replacement: $1 + # action: labeldrop + + ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertRelabelConfigs + additionalAlertRelabelConfigsSecret: {} + # name: + # key: + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. + ## https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. + ## This section is experimental, it may change significantly without deprecation notice in any release. + ## This is experimental and may change significantly without backward compatibility in any release. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#thanosspec + ## + thanos: {} + # secretProviderClass: + # provider: gcp + # parameters: + # secrets: | + # - resourceName: "projects/$PROJECT_ID/secrets/testsecret/versions/latest" + # fileName: "objstore.yaml" + ## ObjectStorageConfig configures object storage in Thanos. + # objectStorageConfig: + # # use existing secret, if configured, objectStorageConfig.secret will not be used + # existingSecret: {} + # # name: "" + # # key: "" + # # will render objectStorageConfig secret data and configure it to be used by Thanos custom resource, + # # ignored when prometheusspec.thanos.objectStorageConfig.existingSecret is set + # # https://thanos.io/tip/thanos/storage.md/#s3 + # secret: {} + # # type: S3 + # # config: + # # bucket: "" + # # endpoint: "" + # # region: "" + # # access_key: "" + # # secret_key: "" + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. + ## if using proxy extraContainer update targetPort with proxy container port + containers: | + - name: prometheus-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.proxy.image.repository }}:{{ .Values.prometheus.prometheusSpec.proxy.image.tag }}" + ports: + - containerPort: 8081 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: prometheus-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## PortName to use for Prometheus. + ## + portName: "http-web" + + ## ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files + ## on the file system of the Prometheus container e.g. bearer token files. + arbitraryFSAccessThroughSMs: false + + ## OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor + ## or PodMonitor to true, this overrides honor_labels to false. + overrideHonorLabels: false + + ## OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs. + overrideHonorTimestamps: false + + ## When ignoreNamespaceSelectors is set to true, namespaceSelector from all PodMonitor, ServiceMonitor and Probe objects will be ignored, + ## they will only discover targets within the namespace of the PodMonitor, ServiceMonitor and Probe object, + ## and servicemonitors will be installed in the default service namespace. + ## Defaults to false. + ignoreNamespaceSelectors: true + + ## EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. + ## The label value will always be the namespace of the object that is being created. + ## Disabled by default + enforcedNamespaceLabel: "" + + ## PrometheusRulesExcludedFromEnforce - list of prometheus rules to be excluded from enforcing of adding namespace labels. + ## Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair + ## Deprecated, use `excludedFromEnforcement` instead + prometheusRulesExcludedFromEnforce: [] + + ## ExcludedFromEnforcement - list of object references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + ## to be excluded from enforcing a namespace label of origin. + ## Works only if enforcedNamespaceLabel set to true. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#objectreference + excludedFromEnforcement: [] + + ## QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, + ## and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such + ## as /dev/stdout to log querie information to the default Prometheus log stream. This is only available in versions + ## of Prometheus >= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/) + queryLogFile: false + + # Use to set global sample_limit for Prometheus. This act as default SampleLimit for ServiceMonitor or/and PodMonitor. + # Set to 'false' to disable global sample_limit. or set to a number to override the default value. + sampleLimit: false + + # EnforcedKeepDroppedTargetsLimit defines on the number of targets dropped by relabeling that will be kept in memory. + # The value overrides any spec.keepDroppedTargets set by ServiceMonitor, PodMonitor, Probe objects unless spec.keepDroppedTargets + # is greater than zero and less than spec.enforcedKeepDroppedTargets. 0 means no limit. + enforcedKeepDroppedTargets: 0 + + ## EnforcedSampleLimit defines global limit on number of scraped samples that will be accepted. This overrides any SampleLimit + ## set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the SampleLimit to keep overall + ## number of samples/series under the desired limit. Note that if SampleLimit is lower that value will be taken instead. + enforcedSampleLimit: false + + ## EnforcedTargetLimit defines a global limit on the number of scraped targets. This overrides any TargetLimit set + ## per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the TargetLimit to keep the overall + ## number of targets under the desired limit. Note that if TargetLimit is lower, that value will be taken instead, except + ## if either value is zero, in which case the non-zero value will be used. If both values are zero, no limit is enforced. + enforcedTargetLimit: false + + + ## Per-scrape limit on number of labels that will be accepted for a sample. If more than this number of labels are present + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelLimit: false + + ## Per-scrape limit on length of labels name that will be accepted for a sample. If a label name is longer than this number + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelNameLengthLimit: false + + ## Per-scrape limit on length of labels value that will be accepted for a sample. If a label value is longer than this + ## number post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus + ## versions 2.27.0 and newer. + enforcedLabelValueLengthLimit: false + + ## AllowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. This is still experimental + ## in Prometheus so it may change in any upcoming release. + allowOverlappingBlocks: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + # Use the host's network namespace if true. Make sure to understand the security implications if you want to enable it. + # When hostNetwork is enabled, this will set dnsPolicy to ClusterFirstWithHostNet automatically. + hostNetwork: false + + # HostAlias holds the mapping between IP and hostnames that will be injected + # as an entry in the pod’s hosts file. + hostAliases: [] + # - ip: 10.10.0.100 + # hostnames: + # - a1.app.local + # - b1.app.local + + ## TracingConfig configures tracing in Prometheus. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheustracingconfig + tracingConfig: {} + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use prometheus.prometheusSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## Defines the maximum time that the `prometheus` container's startup probe + ## will wait before being considered failed. The startup probe will return + ## success after the WAL replay is complete. If set, the value should be + ## greater than 60 (seconds). Otherwise it will be equal to 600 seconds (15 + ## minutes). + maximumStartupDurationSeconds: 0 + + additionalRulesForClusterRole: [] + # - apiGroups: [ "" ] + # resources: + # - nodes/proxy + # verbs: [ "get", "list", "watch" ] + + additionalServiceMonitors: [] + ## Name of the ServiceMonitor to create + ## + # - name: "" + + ## Additional labels to set used for the ServiceMonitorSelector. Together with standard labels from + ## the chart + ## + # additionalLabels: {} + + ## Service label for use in assembling a job name of the form
    2t=zI(Z=NTeKK^m^s^L%d5$gIrco!D5}@B6N5uy0bu9pJf}XX*3J6ikcXNGYPx`_f#H?b)RtzTCOOhA6cgJ{M49 zw&07{-V1&25_jb2FrqRS*%se&9ns`EYX?qf>;V!(v0IN!Uy~`5CX#OP?Wqp!Rhf1m zZUV9z(iFG55DTNY`ZoNu#i;#t#1#DX24}&$LhoiKw_)ZYI%_zA%nmzTaW7@ntW8a1 z$?FxNAB(DkvkibvdyW`3hiWlx^MOB@9!+Wc_+Ak5vYzHqI97Rp353EW2yO%-BdUp0 z?`TbkcA8RU{;16g9u42v7U0&oE&HYZ5IvM5M9r^j7(EBHNSr};g!z30<@9qxUBzq} zax)RZxsdY0Y(H`5HL^?xv^N=kF=m9R7!3wu@H<<1h*DtmmDNjxM4iLD?B75U;nuoj z+ekt;e$CHY?`o)QZxUF>@^+I{jj+BvPDY_xH=_!uhJXNnR}8^+pJG~GWKMtF3L>1q z)5Rcl24aW>t7_|2znU)Tz1DAT2uZNca{|7qNh_5qCJs0&)Q?_MOo^Rky6XHf85MuL ze4|W9(k6Dd(qb%Qi`vUpV_pgyvJShA!pU>^SexZy-^ak%J`-}M<-vF3WOD5~LPNy1 z^WVB3&;R(G#8(u5<~=w^hn^xPWa3$hxW?JCF!CBBYAde&O>y(k94$@A#&5H29v5!T z4of1&e%|(O-UkxKdVX$Sb`WV(j>Fv!|2ZawaV#Si#Lgv_WkDWf*ssz1zcpx|{2;LPYG9ZW7sPOrC%5UsN_%7Jr4 zoQ)XQq8>lYdKuHG7d|MC*t$+3=Ts$%>`_9V8|t`}AsVVTfa6#^ZX?jGssW!lH*&X1S!EOT|4U3}Z-!F~rmZ zF^iN;IgMn}y~df;oC$L)rECln;*5#(HxGZQB{f%6uosU@*<>Z8N>h_tW!dN((pC@8 z@0Bxn^Q+hAXv+XJ0G=iaKRTC|8Rs$w7tCKR$8t(Fzd;#PE6sj$Z&FIf*hrk)js}gA z5DkQMwFm|y-*EcG>n^+g<7noaT7Zyuap96sxB}cPuJBqAjRr#uf}T zy2#Ox%s8krdJ(r1{u)UxS~opZF7|g+jo6;WsMXq&EC;SoUjvS-jr#|R2X#k*FYJsTOfeLYV8CLJF|h;6am zW%8I`zaYI<4-D{IWStD5H@s}Z>@rsTq%)*lF6lI&@f{e7y(}WR*kKDN4%Io)0-K#0&6ivw&t~ zJJLu>Koe~xD>?e({G6njpktp(tZBPQ9z=NM1TxsyTf$V)*X3g>SR^Cn(D{X(u&WqL zOs#>(VNR)NH!Ffim7cG8aVbhiF?-mFN{k5&UTTVZIG!zbx1V6J-1)rbX=4hp-vNa_ zMJIjy8mIbRekb^z?e>nuPW(Ue07Rlpz%zkh9lOmJ9H+PcQdaXnkJC^8t!;k=f^`Ig zbv*5Gv+YY2(KRwLs^VY$@H>obtyUx*2-kZA8}C3-PnYb-^}6kTIfm8*!EXd)9R$~;mXcpp zznh7iVA69nl%mJURC}Z(T-~ruOyQ(~a}I1fmC|qh?!w}>0lsQ7T5dOkSZ$|Iu0_Awf?Hpj3pCI!Dh=dTnHU+6M0si(XX zc_AIe&i&mJc#02B)iCiyKtk4u>D!B$OPPMcA^652=ynu<3ITS!k7l>87(28Azs5aO zbR4+x)yYSLQz7b1t0$j-8eXn<95+tifIg-Wf_Ur0)kR4DnwksuGW!j3( zsbFmDf4=DpJh;N$$-fAO_}IhUH;gfEI%NpQRDhBDy53#}jkM?JiBj5Tl5_`vV)uunR_ac9h8KX15>+Muu z5TRNi_EN+n{pqk3YBbQ@=kWLGw!e?0QdL_T0PgW7fSQ_4b|=S`oD$=Bt|h$bpbPU# z0+yyew6esUGsi;Rc5;5I8n)cOyx#5oUq0WBnVqt?RY^JXrYA#Ms!lTAvP`8sR{+AW zlZTn1*Qqu2mpsYw)ENCk+F674SsO`g84Fh7b_tELd>d4uUBWus*0Vj5RM6zbrk{JZ zS8G=GzsC8r$3!I!Z>8L2lV)1#CB}&!o}%vwCw967I^Xsa5Zt%Uuee1Tvc-h-ip6YI zNbRx8Ps9*j>H9DhJW$w>?Eo`eENt#Q%jl*ieMu56(cUeLe6?Sn7_4u9lBMWJoBGtI zRPhme_()WOTmQ&`h}aFq1m%*qLeks4kDEitnEo*Q@!Yh5o(F9rT+P>VAkUgtaN{Mh z5_u%f3ZkiQd*5?CAc-YOF7Fp`=wUP&OwKb~sAQdcY+vic+Fv6o5GvS6;W2D(_(1$y zFkNiEAi0(J_hV0Tq2X+;>RXh9T2t)89qIO@MK@nl|8xnWDc4PKf#W9?+VI-HCi!8T z*w6-61UG41TEvNdY~vJJ_ikllukmPrV@OoLgXnmc^!;^-9EP@=Ml=8@mO9+IUXyB} z-(}>T({NmQr3wR`{5Q?2H?WD$52I8yD-^-k?EetVsdK!U!rg492mVFHsthXlQB{x`-OC4LXw#qT)qPBU7QXt?muXv3 zUzE~h=AM@N9#Z4>v5RX&meJFi;ckUy3c(w!ZcNO*!&EQW$-RPYE|!TS#SNINbKz>) z$0tgBU;`FcBeDOH&!D ziIQl|djeSMV5oD-A@WRc+HhC*=z0JT-ks>dRko#QHQwkP^;(~Oz?|kd{vnpj1`Yd$ zK1zk-5kL8(+&ubz+w&c4%cu=qy{8=ZNL$%dj7V0#T*_uZCGh*rD*}+6yCTCB))WxW zuw;OIJ$k%b=qTz*avs9G=5HtfNU1+miREwx@#RakV_79y1#QO9Q-Aq%42>E!YZJ3f zZ8E>WGYiqyQX%Uw5j5uXbZDyoKvR&O{!ns|Y$25;>3=WSZt0y*UM9;f+6Q9}{hG=d zcHD_$s8hUWN2UGTp}_W%ICKfrSljM)baQiiKEl>d{rO({e7RhhnertTc>k=)@qH_i z%<+92(*I9=U?%!HyPQjjnHKy&=iS&q;4g@UixA4n+RJpV8}6Es`gh`toXN&E{^LV( za4@e$c6)IaP;SaXPl3vb=#qY`R)sb;i26FcQ#Dhp@53v=M`_6~Lf7pnE-P}W zA}8%{?0ZP}W)T41^bnP3dm3|Pl!nTpg~GWc=A%B#Kndp8p-?kZ93$8u-3AzH)T%D> zTCV&dzgQ4)6VICX`hzYgfI4~zjBF=njol1tx*ZXx^^sK^ff~@CY}PrO8JN-T`l^24 zVzb)f{i_a>*%pQe)^RA(RVr96#LDKIhrsG`bwIqP9S}1IztW@QOx=Nf`#g9(u)Zot zvQjMzzQ<#DqLCux<+kBGQr?_)d(Q2GF+4V zn$%WU*>*t9r>-^ zpInVoNDH+X09$5-uu+dp&AI0p2mx9F?6bPflvvvPrK@;kr_rxb5unt>^Djcg{NFN@ zf=WHrT3KNPEY?^&U3{rXtV-qrtZY_V7&s`KZ`y$KKScQ3UqS!O4?(T;)uq9=k*CUshC#r4#=T)d}_0 zu z(HO>pSj!-d#8(P$SU)FcXQ1_sr4n+GAA#wFd+8MoP7KRdGv0EwB>6RuIzb_2&L87c zs-zvGE7!s*!>E(0ls`vLQ*}w2piUz?fI+7E^Zd91T0r9keY=#6K8?`OWL;uS+o|wS zRes;Gq3=K?9qXLv2Vp5w|Bf7n#JG^M50}LF4v{FWFKLs=a)@me@P=r6pvygX7^qKt z&_#Un)zSJQ2n%yqj_)I>sV^d;ffulEF$7Qk2@{|KKk^h^I->R$EY!YyViZQIsH)C{ zL*|rJyDX&@AvyyUoid(6Oq%!NQD|D0+nJGI3R9XGZ+bz->{{3e8wGQI@ zi-Z>8YU*>e@cY}e1PqsGxERiu_P94Wl*!8TGL$Zz|jpt&&EPw zD^Tk$4_9-QLe4N<7}rWpnrObczEP^53^f1QZ@9*YWVE_t7Gxttt4(z2lcfrd7_ez0+I8lPZbdh;7a2&&avbb{E*)4j?x$$ytXQYXD4E55o2~nO!3K76xTreB zspD7wo}L9i(R>JnG&*WM-#2sJ29D<|C;L?mjO*KI>H^`fvHz1|I%qc+jXw{0*VlXs zor(FeO>Qhn&s8Qd8lR<`5zYHCcqD9BX%C$chFluKntb^BA zyUlPM%DeV~q^qUfrNP_x?enE6;@Q#zEo1VM1bx;~{}WBm_pBWT-KCO(>HDSoGaTYs z?$J@^@loo*SLy*Q32HA6dM|?DD}n$P26Yq!edGu6M^R=^RKW|WweDH3Z2f?jf6we_PD`k(lCn#>`Q@q z;}ujOLYEBAs=Z0Yt~c|p@W0!Ee>0+6;%?XUXiGd4{5_#|q#ZA?j#0bmg_`wqP3}`^=jt` zd3k{juR(jj$72|Rh3F*`KR}4AaY#CMD)q(Cfm5HkgWmA3il4T_8aYz=ZA*nxwNOcH zs^^V9*AyK2?O&H#!3iC>z6JJd*x@eo>)QN>^{GwyvkHWv`h=kiTEpfg^bAiuwy$q4 z4dcyGE7@Pit%?&0;2CCsoQ`zHogPmtL-bQkgVkn?P{u~@_JYlcYr`I3T10udF>-UW zmC@SNuU%wh%66~zh8Ujw`*V2?jR8~FeQ5_sc!#Z$04tMX(^l2xk5f0N1JH33@oRui zSY;oF0{QtPVi^sN1~I)3;ec!zb+lP;qq`zaS*SYAX)jtv#s9jQOyEK#xu4AcGa0=G`-P~o{wm}H%sfV=#fCq~P zyzCWv|9jxM<1aOq5c$J7o1rJE6xM}o@D&e$=1`A25LFY{O3xX4Be4Y>$ zMcWZU=DkU}HmQ5l%gzXPRRHjNx02F1om;MqR>1B2J)sNaO{Zt`0<<^dU;G^S=Yy*B zyY}J^ffc)-Dvu8QyX_Kq`vPeD6SglBtT()04X2^hljM9 zX=Q3(Q_Q^kM8#bJNwgw^bqbuRiS&fVQQ}7ojqK+v?@MTuc%02W9KTx)F%{&22mvnn?W!@F91aj6iox(3iH4Uwxz&-wNP8U zu~bc{0dlCKQ2JG4WB&pYM=8(!cSf#5(l$-aZ`TZ)``?vMs|ikuNspX`V-3;c^qN#O zG!X1#XeKJO|KQtmwQZYS21BMS;Zj8Py|JsnwTsncmgs((-~`^5z&@9{mv@+k=_*M` zsZ)hh`pkYVJe&HJpKjj*O zcJv(~K>xwC+%>eR4#PGvCmO#=b>5e-+2HX-l6J*WYNJ^?K#on=AO3b$*<59jq=a$j z+j2_L!Q}Gdp|=Zi(kMUq|pUR0KiaI~nZ5bR`K zPA$sAX(tw849)vqR=SWVaF%so49gV-eNytS+7BUjGs*`;@ zPZmo_)>7i^ea9o%*Q8daDeB~H#QiP+cDWV0J!*gyi_dog2B|@iK+o=_i`pidC4-+R ziXo|X=n|rnf7_g7j*_E-0f9Ho<3uj4ze5R43xl>)pP26)N0y5;rS4?>Qc1}L+OPyD6UMM%Bb$Xi@Trp5aG z3nh3K()UYCcs-~ZR}$7BE-d8XN?^FpXmy>03GOJ6!V|FB_u)pd^$*!wff)f zW+V)esuJ`lIJ3=utcc5WB5HRkM%%u&*rqkt@`PB8>VqL~DoLorE}d4F5IpvwT-~y% z`8rU>ejmpA68-CYv$3oxYjkPY#IIL&S!{K`9Ibuk#l}L}|BO`2*mMf!<6Bel=8LJ6 zOMRJu-a0yoLsI0I`toK|JryboCP&Wixa4+KLkQ8(Ce{F{@xBFkWO?nc0!dI4jNiN? z&?}_6|26AK%A)1oNctkJUHu`RwF&M}asi%lmz(KuF;IbAtK%{dDdm$M^13 z!8}LM^YN&}SHv8sUEq$PY zywE-^{YRp@AbhGH-#}lP95bhC|GL<1_+y2188Up+%q=?NUNN6b8@C8y{yR49^3CRA z=D0&)sw3)VS+u{C7z?5Vf5#kC{a(Y1yyn@5qqM|Te*^WJ1;+~-+E!4I%pU*zY* zw_rd+OvZto{A}-z`FU@-WE1@BgW#vlxur6@CdoJEJ(V11K@24qlgmZCu)m@c)@2SN zrM-TwD)7CIY7tf@JGS@wHfrJO4MJSCEp}?xa;Hc{MY75b3K4i zzgG9)F$+zv{(JLZ`PH|jBjg-iE}yOI)Y(FuCc1{Ji4f&@b1(2dGRjIj*D{G4{Cstl zrrCrB_paPh!+Y+iEPS^P%H2Dj!mvPoPVp5IcWJrxaE7H%L%E=U2kl?) z09b%JE`7L-7ObBRi%o%of(?I4ckVlVsR4%v)9)rxHf_bDSDpX;aziGIoB!Jv$u2gL zt5=c2MrVkpRHIP(M^eIFf}YCDX2@LeXdL2sUn?5C74PM5>LLG|4Kbrqe}cK1bKP>j zhN;?~u5u-0ZrgkHMSI!Z6~c3XCSzbJujzc5$)ljSyv1+asCiyH;mQp>?fO(|EwXU* zNms7qG(L~oL%g0^)2$@u>H*|Yej;ppwUxYdg7GS(B@vZc@G(oH0WxxFoUkQ5j#5vB zY4Zl?EWOM5Vw-5a3^@DSBv8S1w&nQqh(?`Sgf0q~0bI5B`ag{PQ*bZ;*98p5C$??d z&WUZ?PEPWTZQHhO+qP}n#?=4!yfsr(H5YT!pS88RFLte}-o2~Ki^3e|lS@{lPAqw^ znd?av{pQhVgN6$j*GnE}p3J&qLz9;r&tzmomJW_t(+$dPw?#@KsfNm{0rG?w2_;m9 z7CHODPW{sXY1lW3u(!u>t7nPpU(}Az^e3b86|#fRXQb~W8ujV*g`<4QMtw$ z&mtT~Az_Fgku42gBN#9VH(N0WTh*A>=fae(-`cK(ms9|x2GKFx_7mnW>S#e8^2Ajx{Ex+fv|j8LMj~a7JNO#l5mOA z^p%A4jVV2DYtRqhx2oSx101~rXZ={8G3nuF4x4F6{@leNc2|;PC_=)U$?R=o7yhfd z$(h@?CHG5v6XQrYJh~$5zZ8x$q~N{C;xt~X&3Fxr>jx^>yJM)`dvg)6mCUxnk(pq^ zUTtRjN3t`;bn7$4MJySL%iBkY(_sCsWn&m1r6TOMNXnD1fR2(C6S}dCgw-EbJwfJz zcI?B6CnQ2xbYNvOT*Z_-)fhBdFVEJFG)j?=q{eC*8%I-T)y|-p1Sp@0-OJ;S(9p_9 z2NK}|k}hpvkG0wI9r4%|{BBKu?|aXBmsbP#ZxtVc(wbFb>8IA<46Mz2^h)};)toSeljlnjIcg8sTt4GB__JFJ|WVsv-HH?P@5Ug2fo3c z+Xldk@AMD>Buu#kBVC)1$FQ!%e#qmX3`XvSQEwOYWmU?0Y6Qf5J57LxCAV=_5g3ea+i z$x;h6VRDJJUi&G29Q&rSuNDgNn;{Wo-9OUExpmHc86f z@PL`5xl4op2J6k9EYt*QQXgnz_#ZjSC4z?b;W};h%rzKwM`yzzT)%bt`R86Kd+~=h zUO9v@GF{qjuFjfj*{OE%84YDYBQwJx4SO`ks;brs4CjEl zB;I@jqKH0JmW(s=7Oz}q#Za`7Yn=*dZyWk{IV)m=0OArNT&dJsy!#cuhv=P0%hkc? zfH135oSFz6s0KxzSNvwcG-d#lnaIjGn=uc}OOC0-G^QHuYLx#n*L>eVyIE6(*H*7p zO}3}b^%gKngErq*K& zO+ODEkdsxy!X4Q!5v zuYcospdmMgi2p2U&Jyt@n;s$)oGKAMP=jhMXAgRPwj!@Ii*e{Adl7d%Ry|w9bXd8{ zRSvEbXh9|VDnYC^8KlszpXjM~oPU+AWs<)bmc{=rWsXtIbCjUb%!t^gd3T3QmAF7| zsZC}qQQ+2bOxNPZ<~G{1I^L(YG`9BYO?!tQa@&5EGaV=3W-tNxviRBA!FA1QHcyT6 zg)ZzmHmneX=UqBf2U8#DtYWex=GXoO5lt9e$QkTj1FJ-vi>Tr^g-8F_A^ROn> z!=r^;<(z4&kn~vANlbObj9xp^D5a8Alp7k# zBNlhG`W)a;KXU1BeXYl$pDu@Y&|2=ch6CB1afNz#d|)Nu-UV;c7&UNZ3iCt%+670# z8Cn%l+0fOzrYe!eXXcHPQsx!aM@5_>3Xee3bTRtj28-XaSBHB^0!Tze$18?nqq+d< z^Iu|G{L`S-XRc5;C+DK%K2<^yoWgo+y^7*v5t43s6f3Qf23aL=%{#FR&I6KZYd0pyF(Qw+3oN=^z zO>GG0Lhx)ix!DmJo-NWAcZP}J*y8eEkTiLsdcL$z8vG+`qpbB3*WuL-HZ_(QGvR{P zn0oOS*LzB=rp-Bi{2FWK?K~@Jr9cI5Onikp*Yf6W-1xT8gCRBx=Lj=VvHztC4caPO zgeC_PqOMwPMtd}$>~8@=S+-NQrh=lE>Id3OxPQfAFzo$*C~`WDsf$*Kqqf`N2eCY@ zaw8!s%|*x--kL_R3@1kTwW2jBDU`3YyKR=lk*2akHAvl~{)YrCkhk zBD7h{lJ@gtUxB}>TM>6(&XD~>!&`VX%?k;3w{Ie_)lYdEux^fhcHFil=lt-r%A;2W zZ=Nny`RI(_7}*goXB6JxH{5Azlg(FRicqK}_p8 zI(15x#;oT(j4og{f4)Bw`y3B~sOI?Wa+i~@wS&kyz|%AGNq~I3v~AV1Vht=iLoWx3 zGCH+k60X*+Im7rUl-QX7IOT%k&|Mxpv5d-3ZB1b^NPFm%a#ZG_6I1x}oS|x-#5XEZ z9O?C9IJz8Ujyfg)9IF|PMG1;6jmC~;?-QLV87Q6M=_KQ59G04t>&jwDZBUAmGu}kr zn3EO*GE-YiZ}t3iGf<~3-eP`e%Za-MCBK;mvYK@vy5lGwuYo(*sE+A(y@m{`(kiF6 zK5B)fkr@^}3!Jf)&~Pd^?rg)H^lWKz^JN#ykn_9mZyUG2XUdm3SyaA9tIJ_hd}(d z%C*s;Sq*Nvh?XvLf}=_-ICMqX_M$`GInSEF&-}0rx+q+A#U!N#Wt6&1^Xgb1X^NEF z{vIu~A;5vL8X5{7`jz)X;MY;7qrg68nVg-wP|SSkEDAby9t+&qNi2G%Y#xh^yKoCL zTd|PE%1MlE^ek#)_wdzGOx0?}fULa+uwh}lc!NZfF=XxN9K7*0S<5tPIVM}poyR2Z zUu8WQy>_UiUvs_<=m2cWF>MDr=v1s}&wYD5{xQZ^YH68t1$;=-kZzQitesS|D47C^ z&c?8mz5OSa>+qm(fYJ&Qk)_bcvmROD$FWrhu={&W(IwL{6%EC4_r1GG3+nZ#b#w88 z$dJJUf6TMiyTT>%G7@lTYP;3Nb_1IO7u2)WztA~}fo!pD-HB#&q@~p^p-pRas@8GW zs&&s%b^GWlWfgu^bE#P4BT4Q+8Tr&WGhx>nEzIhL$vRg(YfV-oc7FSy*G2!d#x7q? zVOFlM&sUkLbFG1xUh^VW2DsLnUeCpD+gfZZ^x^lRXq!ZwXeLLPmZW0A^lzQq5AS&K z3ES|NK3^JVeo_;YD%fYfhBQtk)ja-n5(7^ZSzE`MjyA+nmY;-pP7c*8AepvkjDFVT zxi}@`+uYhryImQQEGUr#j1R;f)$5$ zpM;_j5e*WhkObeAp<2N}XB+EFo&?>s5#k%48Q--!kF2^EnZrIE!r|;ghc`i77?753 zcd)I&uFqJmoE1cuL8?c(vy^XpRN_X7p7B>D3w*^w`O@x*kBwqYujwnd!7DV?WI51B@it4R^$!)Tvbl!)Yp$j)yfWl7d9=8f#ZVy8(|zS`&O^M14R#snY>H))2twg11S`T1WD#m3SHRx|z2Dfs)p zyYm4qjIY93UWTwd4^$2ZFTxMsNoLAPjbw*VW=@>yyK_d0xp{j#x@!3Txhx}nM#uZ4 zZY@yhl#yv6HDAg7TNn^bK(t&sy7g105+dd9?GQ0j%410*T%ze`qg2AI7oRX%mb^k+ z9Sfk;*u&!|(%`d+lDN)X z6mpS}WJI#3#fG#d<4yl@O1c*l$s!LftE7#mcXa@(8GNjY-_X8SNhLHJX>fBEMOfty zE{m7MgM6nhnxND@37X>cCRCWJi}(~vBqKZT2+u|A_=8E8JaK%uxbiIJPArcygPcI) zUUB=Ak+51?+ul}a!!*xpYeZSZ8yqO`jroJJmYSI=ug@=bYH7vejFvDJImOCO)F1jzD5YcUCzw9W;Ys&U?;s>;=6o4s^6~;}kgB88z*;u>xA#2M z={VYlEEu{wUGuqn*$ZLvPlGtdig)!r6 zOm~|@8iYP$7|9?49&M+g?m7zYgT81dlOhrlymy6j35k%(q8v=Bn^^&)e96&6kH0BQ zB#pv%n8Ha{QKV}b0Usj&>V7e~uui%9g@`8<5|C)gBf1ooolLpOF%lX?wQDke+W*jx z^|cckoHjQ)8CC>Ty=`zZGG<^PGE+*3)sgZI=mCXl6B|!Ku&d;i(8(xkGJg11&tVaK zhAC5#jAo{36#x+SQqIp^xSixZ_d2p)axdjh>y8>qx#f)?dueLnEpRwx!Q&T#NHc-V z_Y$$O@4EGL_a7>}+-IWiai`f!RO<81Bra{j&?+@|p&(L5@q+7bC0xe}oFb~9FsN zzx_nRnvm`5+0a;=En0PoAn!=rN8#r8dJJ7z40GQQ<9<3*EsCh&af~(>vQP(EwAdJ) z%*rB_W(9^$8Dpc+jl`~8b8ZWQAop@6KhVyBB)RChUZl0bgBD@5qg&Id{%y=4ObJIFc zetO!ycFNPx4sYKGL~*I5u0aVu+#9q2;#W@(acaTGF67GVbjyj|Hu4xTItTHzRPDvumBs!a04bRAS=&0 z5#c_IrJ{{fP@T8=Hv~13rr*5i_pXj``a$?*grX<|Pr)F(&5-VIjI4g}%@^7CoS`0b z-eAP+S_@fFHMs^4hVbpbPY>j@lq$H^gtNu-Q+=6PnF4C}?c=1`dL@y%(X+GOnH~IZC-(PP{hzPJr_{sR zyQo{tBa$ia?w_^T2}o}&u97X*558R#h>PGzW{Hu4?KHA!BP>jQnYAZI>U)YcCXfE z=Y$R57UBkN-z!yyfYlh7Q(hP;!3h3y&eR>(0{8V_e>#1m+JC4kRo=4P(a>|$lGS=0F-t2fXy~UY! zp;y#f-qNxbIZ%XyF{a~6kH5vhbaQCR6sDCtGJ<9^vO}UT>fGp(s;kBQrf6K@$^b`pt zr2PkKvuDQ{kb?V+*7~JdWeO^BHi2&WCIb42P_u&j?$;L-Rg84|e{C|SJDIpDaV`}T zx}+V}?m$F>be6GUHNB{03d_?8-{1|w)3*?_OOr|TC@JUDorVcs^!C-lnYD|+tavq? zYV5&I3Qu3LGc!mE&eX79(1W^*r;Lp>2z`%#Nbbh!G-vkM67@(iY_j&;KMxjy#4mTMVia`m6ox#nI2 zyT+LozlsukCeHB!5Iai(8qv-sQyu3{r&m4t)zCP><9W3cpNeW+DT*pIGcYC+tbFU0 zWIxjKRnJlRj)V2CDRTRu!6YMx5=293ayeJ5DV-Q}s`Zi5_qM9LTWlr%r488O3lB0m z-AP`{|M^)z3Am(6kF-!rRTKrAnG?jDK-E37OKftn@BX_#!`wg)GNHC^%p({;TPMLn z*E8r`oAk6N7~fEZerTl?*}`^xvY-JPKJXxygH6vVSwE1tq07gHd`h}siDq^VwzD8(7N2ups!)beoqaD<~8Ewec(y z;bl7m@0KMCQ4IWjE_T*|{ZAc&PhY>6pAoUQWF?4&;4*?hP$ZAuWo1sH$2xorB6 z;4-Xz{Mn-28|S9D3W(AAkveG=Ob-U7h~ceasDb1(_BCmc-w?zYV=ax~Aqu!$W(vo5|gzhk4H=Ta5I?)8H<8B0Q+rVoj19 z(r0h5P$#uHuiA2pQxYDMsuvxZxsjnWw~7nCKqJK`6NKiGMoEK;XBixltLQa^hK+rN z)!x(r>Lqc9hK=eM41s1CL(|7wmho_NY!cs}v+s0PM=U}Px4BcERT`W9%nVVhE@D`1 zC-W@IY54<5pN-T;_P_6Re(J@QP zZNZTaM}<(sse0;+@F&nDo>211eCgYaMdD@h?{v3iQlQETp0ku9Ec)N+`vyCKiL;1} z9aMG?p#qY}Ny`KbrqwO}ZekjGc8VME1cq^`N)VLtwmujKB_Bd``}Z#3?X)BIFZ*cI z!%zw6{No}U<>kOl9$RYpA%bjGg6To}X)G~zS^xnm*r9f8m6g6Ey_lkjb+v*McYo{X z96E@Ttf`GQ#uPgP2|X|~TZ|!b5>*4_MdY+A#8c%N;zk*r{1{Ae@&^|N4J9w1A8!H& zS8CqJ2G8mNT%HwOIMX-x}i%{V`q1jh| z^q=Uwx3T8vl)z6Pa&n}Cb z>*OpZ<|@Wer}^e-inr8U9E3EaaulvW=xsKU%eNj3U!ZrNL8rkbC8RWj@)ivQE4o>q zXX)2A`bH?}qCPivk=wt_KwpI0?bxujN=2t5VYV7oc{DkcqhZoa{r0lLV8F&MA_N!$fvBJRDX1v32udA>7r@646|fB zVT)#zIzyQ!VQ7XpxYUu;nags}rUC>W27}rEelrE~^ z#STrSH(Vpn@_2^Unygpp3Mn|YiGnBlQwG6=OLwZIYb`n{Zj`y^a;zs9=bB`txk6!*f zN4gL4P>dRy2K!lhF#=Q>mPcJ?<*q4fopOI9R}eszv5xBP6N=n3!P2My*X~#BD&XT- zfPySX<|*~~d~`W?4}-r4m*yjmgA}U%pk&}ZgtS==H!jka8;TnKmB>U3&qx%$$oNU_ zA>18-5oTuTD@_qXeZIs(M04B5cpSnC6RZUpN2=VfT(bRS_NQacpD}mv(j?^=3U8lze3YH{gJ-eL`7eth(zL^ znl&pO+!8hHsT=Nbq(JaTa>>+}$-0p{dmi6TXEn*9Y%f-p@i2;WF6^+ZBy$Uo9 zhBY2cUE;6Nre1cHsk?=ULRM*w%AX}9M(eOaM%JvHf;N-fk2^a@oz|CZbDUdJ5yMig z$N9H(ga=yqeNDs)6tuW?&6GKGiH75XJ&%u04BJ`e2TF$<^_8tK_$6%o5;jhWN|IN( zwVl7Jb1e$XVk6^P+CS465JfBxTx$O7#PFT?=U89QkqrEwMph-~goYvXv-(-T9^Zo~ z5-|~6%bufZf^Lqe&K6n%`^_GzCN!=Z#~kbsa+5sMGBnXEo+GCk1#18*EJ7g1 zmz$XtyB3`?%BusbSiZzgD&~xJVnk_B8Sz{gVAGJeik3TTrBBq71Df1Ake%$eT-R+K zP3Q(yU(hhog7_#^P)BFAx#A?+xc3pG?42FnL|IY~);U{nD$W%-j>=AtM-zYNDDDrA zWzit;wn(Z>XUu2#pAf8d7ccGZ>wqh4VX5aYUY?h4M30EUL(*cv;26|@zehoS9zE6dE?Id)2b*8{1 zN^{u*=i0x739<;9q|)ZE#v&8ScPyyw^Q-76AfSqzsOxysEBp#cu=+fy0$|TfiEFmKu$IHXg5{T_6CSmPublJS6U(Q z&|uB75@79Qg7&hQ>r5cszxwDYytD2&dJP&#yTxI5AOaGPzqc9-V=J%=W38SPqxmc5 z#o0c}#0XTaqyC4Rn>+qyM`{XfGw>iJ>=zp?!~LJr)SU^38)1P`Zd@-CdP^Wb)!L>99*b9!wBwTJJGS zKoE*%TIZO)-^YW?yxc5#!(aJjHcFO7v97O#&C!BYYuqzrP5!!#{n#e$RS2A4e-K}rPnVa@f;0`}Bs z$u-!}@=3BQx%?D0%MW*BC6T2W#+wu3^LcS0869GDB0*_tozo9*@EB({@<~|6WTdxP zQw&~K5^M(C$j%m@ZI3GQK)+v3j|mPmgpZ6}1i2v!yAcPvY1F0?WW7*g-G-D8eTr4% zQP&^y0~$7wNG5rsr}?`aPg;bEgswstxz4&@sT^%`@@s~bn_W|PC$UbR;duS$21dxgW^zq^>-z-3Wrl&GgCe%XH#RusVWV6EU1O&o{BKOTB7 zKC*L++3i$^z$_IL{wlEY?ZZT@9(Bu!1;2vrkzU>tqpD-x=hUxms0RKpL zqmk(-&p#Y`M{%>A$~lM2+!xx$@Vg(V3K8>wHSz)T=MCEUJBW#Y;QQlGVJo-TP-Mda zT;m>G!w%dFXu)3zN-GY*N3ClJ$ec4D^iEIB%Y!c;VQ-(2TnwET1>apyie5nspDaEy ztR0=uY`tnF2kjfW!c_Lcz|^%_wCwJ=j_0iTd1e*+GM zxxu$TzR_v5u|C*k^lwbXj^x&X}m zCR*oVMjrg5x-SlNJ7U~K-@o@K+fX3QOPO}ya%0PKI%B%g)N7?iVq^s@LLl|TA8JWd z3UQFE3=KBMPqShV8RO%A;$IjAQzTYF#LH&t;q>i$A;NA%8~D4;J!FIx5(B6M(K6(c z7#`BBq6q8JR!lh0;Xp@R7*N@!I88&EnjH9ropJnj`~sq@B3O^bhWl9NFdB^P1SED2 z2^MXVVvIBocf4%98+Sr&?sC$gmdgw&W)$(Zx?PYOSg=KAwG6u7a=va7w)8be6;;c7 zv*7xROB(vyNJ2Ei3<_n<`U@Z<>p}705c31bl7WdQI3su?N@YC+34!iAV}!cOi;4=l zb;1QCBS@hNB>{@fP+0TF4GNUUwd?iGKMLWD8A!p_{$eR(5+ay6rbGj9_23-Yp}{9$ zl|qP3eG#xJT2nip^L`Tz6o7uxduX8gd_a5wye^VnS@*d_38*U?VgCN-eIQ0P(4siH zNvZvhT`eVed&aEG#%4e-x`UQWaZp%aDMO8~x69A-aHCIBXWw{(9zhBbEGxLhFmf%_ zK$*c)FU%zZ54EAkmi#=MOJD*9Yd+r68~9FLJ%kQ&nNBa6;k{4#fX)8M*dyVj;N9V3 zD#j{eQaa3-J3AB9o$DJ2*vkt@^YvZHS(fdT^a-_MI5RaGaET3F=nR>wJ$vx)bLXmk z1~i>f3Re0u7x#*BkOKFl6;6LxfJC%j+pN2lD^MEJnhe}&ij;vUI;$WJF*?|>2AOoz z08WxUQN$KW#d)$}aOoKi{oKU1qjz&t8({kyzM9_cJ6B~QU~Ey`-1qYdCKmULP}X1~ zkJ&zVwP3h@Q{A!WgV;SRxX31c8PKrXfV>G=lCk1vhJaS=Y8Bbd-nFt0uN$&uh>k+n zOYaQ72}55?3@0~-^}*v2_yaFBrT?uY?Pc+|s>eKo8(}S%;F)gKE`O4yu1tXhk>o)P zDL)ve)6Ua!HD93V6qJ<;KlQN>n0)|*IF=D;0k;ml_P4*-!;nF|@x}YOI=O@tI4rue zG#I>`dMK6h7;)dgXgY;;FJfE3A^1BdICM=#N&m#>xk$MX5FS z8?B9o>pz8A{H0g7-o9Z^MRDCa)f`E*SMDnzJnFITDW<5?LW7=VW)@9F43FZ383Ig0zNuH& zZ{$>S=n)R^ZH)gIRLUN)7-IKxk%Tdlv45!4xOr3~l-xV@sp8x|Mm@Uz;}NTH`x0xD zs(On_(KP%4rAS8FUpVzFOhCqDjRN! zHc67tJY(*=;Hrf49EsuLXX3F<=Etu>PWaSX_*xM#ONz2XKT#u}l2q9X+t`EB+HFfkgkdLSnvZBF#^t+k2*C#ec*i7$VNNNy@#91>)4-Y*Y(rO;4HSzwce$y!cC z(_Z#m?gj3bOB9&|n)o~gT_3DIMw zgLmcMg2C}?Lq9u*>e>M-60LrG5oP`7p}yeYkd~?Gt#DE_;~DN+K~}I zEyN6r^GvE-<`2jV{sM~(#AN#ARhHJHtum~3_E7S_{mY=kEwB0*uh*u8qLYGXY<@7b zmAjf&VJlq-@XchY1Mu>17M9F3E6|ATl&w^ioGk=#<+A2N-rBaWWsnTFzqGuLIwH_U z2xn9Q`>l@zoA{xC(bZRp2AE_*BX~x~36_f|6d|hE=Gd2vW$ID*Bwpxq=v&Qs|3SRp zIx2Ygiqp;pG6!Qmc!C2F6hbzQ0V|^}8>xq2#uU}?s|r0>C!hPkQFyCZ;cqV=#|Og5vzG#-LZ|#X3!wJqL4YeQYT2iTa%wyVQS9 zCCxB*LxN{-HLG^XY@8_GYNaCbEkN%_f*{uCTDrveW%6)>ZHp$O-v5kF$i*K__I2=i zzI+y__2u*Z_<8{M=tl!ovrAi&P+fQOr%S{xFfkrGK)cX{&6`vC(skFs3QIouBg&Hn zf}LV&kwc(%RcpL{ME0dEJh7tja-g{MBd{qKxhBhNM}z9tpk{q{z1Ii#!5#>Ht972Z zZ}3Mg_5t3qk44^N2TubdRSOpl-8W7xr4~khf^@`qNQZd=d)Dw0PR|b?55G7MrY7+Y zz7XSYGN94vfiKU=)w+x~SKo<6(lf{eX`dAa)6oCWV4Du8(hDlf2&+yqoq@Fxi}cMS z0w$CaIY5Y56ddvt!^`BlPB;_!RLDn|4cE78RLEmataskCFlhOyTfIPwg8-&v;@@m# z3#4uVQE1e4uJqXjf|bptL ztOZP`;w14cTXlrwhJJj&n=w5qKi8E~?nzjX9+e-pn>?F+bIvH>JgK|TdTv~EDSpq_ z$I8(ZS|H%TVN~`06I|M4%)&jYB$U3jb`qCzDgcKwt{uWGNqmw-Ee<8*x%!7)RAGZjv0u;_p0bFSIL%#vr(wvr4FVNh**+2T=VzjlWR9X*7t=V^;1uC1 z#vQfP7JUDBhQ!j6CJS*OfF7sKVS!`@#Jit-DhlkZsgL%!vRcwgFpV|BL>?!I65?45 zcyB-IFP3KF-}PCSV*BrqoM0X+Ab)0FlJw|s*8X`zlzze_JY>F+MnlcFS~oEn{s`?P zhP-TsBdxa#Bg;o{3tp)ru2Bd;|Sz zT#J%_Lw$S3dD*gmHI@?Q>X=2AO$)yDepN`u%`-(lra3)H zb9wu*mhcb=$gK!0K98xnX5D7?T$zwA#w4aNWiVz8#@wW=YLro4J&hB(o*V_k=xY@m zSx%c}aOTijhXrl-{RkysasFbqZ8=t@P)Nj0nb2&(voF~)?o&L9|W8kH_ z0Av!}PUNpr5NQ7I7L=^$83>a>Z)XW~muWMtj0uJJu~YFKw6Opg5EcFq)39{%zWhqo zw$N}Jj6_{}_J>kmp9DE}eik-4Gn5=jd}rVYu>^t$TgJ!KM8!NW^Np@SFz;W&HOwO) zDC^Q=js;rRyAGm_EyJ!Q->_dKd4#Mcb9-dPL!tZf_|aW zXtJq!ifuZ7AH;?GmXCLVyg6h?u6B6J;}kOPhw|m&lnZWkloUmVOSdZStzO zkN1ujUK3vCsnFgnWGM}`Ddm+=?5au2(HA_PyY`_BsqUC^Kjq;jMzRDL^LiCqx?MK>Vhx)Azw1tRt`SXDvW*(DI>c8i5%^QN|~ z45wgQ2qyA9O8LJIQqQB z_%@XNvd$TQQVE2uJ_b;!vJ6eX@k3xqShu?&AT#IaNUqA1SD}~43qW}t>#b>( z!SelPAS5u2E3b@mRSh6xCn?6T2+-85(FB+~)B}I91F2nZY3wDNu_Le3<-~&r{~gtv zBOB$cDu&3yw2Y%}Du^sbVn+&zkRIa_V2Sy31^&l~amI#DGktI00Cr1bMo9f6r6*`wzIJBU!=h zymI^RjY8L@xkEod-ke=|HPe zPzARBj0@xTw<7&EM*zO&Gq9DSUrIm`5i*1eq1W?KHj?T+xwT4jdtJpCgmm;6|IiKO z%iKy%GkoP9Z+l9$VZ@22WilS3(pM2|TaQ8vag(bOh9i+&CnkZU2~IQ>D1So&Ihux0 zk1u%PP=X3jH!B3!G_>OPwR>wSxvqK|EK z(Va)3bnc#59uXN?S4tD2PBKYyyLvPWDM`Wi>aFsbx{4^)WtJ91o=q0nc>rAon<`b5 zDqfB`f*P1W65w=ey|sV(22g}3rmv@oU*?vp%2$CrO-!!%HZyloG7LzP1au~J+sMOa zO>C%MfkTB2jAOjPItJG&4ZOi2^?^Re;wjV8mE)%7~K zELdz9ZDvAt3R3%w1w;w-xe%wga2hP{In);v-`j)3Is(3wN;7w5#x~)8xT)}Uab`Gx zM}f6Rt*a6(DCMiU{B_0Oa38im>r~XNYGl`9Dg=*)bbV&=yi2|Z&pA(fX>??voyq|x z@?UjuM#ys_ap|m2V53b)Vkgc|6vL6U3lSK`bi28R^0R6GjaTtQrKPEtVRs9?O1rhN z>`c8Wu^#u94y(H`jdK!F!Kt-M-iPaFc%7(0cXe)#_Ya4oEC5eHu)iY4nVmUeEVVqW zAV@|9)(NzE`btB9nA-6EUMHfbd6yc15|>VCXX{IOGWuIZQ8Hm#UM57Oyc`b%g}Z49 zPD!aHa`Adfe}$Ju?TAWqaqYx0njtYq^1cQ0#0J z`jpEQ28Ds)&YDHH*5?l+Yme~F!%l}5b|1gmPJPV$)aa zJZ^|1^ugVP4dx=C-r6U%t*qTDl z-hZ86ksR)MdFnKp^rKp-9LQ023;{+mi2~2J!>cCYt6&TL79paHsZszZ+m#TrV4ndI1v2q(JW{<1x3M*mrJZiS2)W;^yrq`rfxQS^Fb489YJdOC!Fhz}kf>{Q_1PRD#GYA>XtC^0 zV-~Q-Lzi#VQoj$QLgx-#Kz`KoJ(Y)ZF!4?@JP{(gw9xY4uql0O{9L&ROGCU|zTKn0 z|5vV6x!8m44}(XD!a15IK-6bxVT8tb?zZhaTSIlQSgR-OTEKuXnJH|!G5b|q9vCb9 zzcHG5UA;g~uLt1$U~{zCad+aCUJi*xm$a6#UM-_P=lyd`nY!oBz)r;u1EzH?H4W_m=>9#^(?; z8Jr4}8ug4yph^VggbLNGh#*^%g|t>%mOUa^TJdOM@~Tm)Xq(0Eb96yS+qTnREubx= zQt|`jGy*H`; z*MpS`Hj8rOewHBkJ}^AQ@&W-LeUkG*s)ZsLXLluZt0MY^Ae`Xbjzu8m7zg9r0I)V(`?wmggpnKMqA_sDZGWbm4{w@@v<{=)gmqvaD;@~8pF0voc0Ry zwyr++$@~L4>$h9h9#2r~`duYp-qR}2mL~8PLYPRY-XaMhFo<4aiAY*BWjmwSRVuVf~}Y)!>dS1^v`UenZSOy zyIr1$jC@(_nQnnwdgD-6J8)|*hyeN+H;9ziYpq80a~1lUAky97yEV`)*>5M0g&g{h zm#O{xD18xKb>`@Ju%2mmzA8t+8&rs1H3O6jL~7m5Nv(=XYn_ZuYUZpU4u)~?5*izy zNDHyaSCKVwLm$lwefADts_3~%kQZKV3>#SzbW+xm7owMaN};!*KHPvl<6Kqq@C&?} zoat|r1js+bnc_AHe7n7oM);UZi7LCHJ6ezc1B7U~oe{ner8yf~ote&ZkE4V(_y&q< zTxd8BNGB;q8)tnUk>Svp!HksiB)VrOZhi^fc`!3JE1*7SiH#LhY@~pY9^{#wB-=kfm7Cf zpGC7dk_g?H$skj&QV0jmJXTMim(xtyvA!X;#u&`Kl$ar`(E@ zapZ_47oF!OMs9!6w1sW_Lb0C49Xp@ZL;dxzj?5ku4$T_$Iacr7hHw(P=V$(uC$`^T zjb2-0tv<)hH$GQ;3J(Iy{FLy8t1aAHa{uB8Lr##SJt1VhdaxWlPpESD1s$;E8ecI- zID(&}oWJUQ$3s-b384A+D^-T9gM40;9zNU3HNgr+#avRh8xJUFJgHGIJ%kkJy1@oU zVHlz=;5?all^Un4+scUh1sE@}OYXbWulKT{om=s)w)lM{t$9F%R9Gua#d<5{ynq|_ z18=0JkZr-b{+*^f1HNWv4Kx$==Kb-eDC@CnJVZLZ8m5^R2PLyimLx=~#`SX(w=}R6 z2O`O$XZLPFe9M&_=1N9ViCG+eeam6i3Up(0Qz@)}?F<{j7DlT(iE$z?gcnINu6atd z!HvXbmN4n71qN)cUi9_amlS5dLE6xIw}cpia5K8Jh{;|mSH2CsVVp!)1a@2&6D> z!(^P(5nghTBIz28mfvOt+_}d!&Rpy|@S9P~LFYJA>te6e<;bHpKL>gA!4&r~^-wi5 za%}Y1nA-SA)#y=(#rQ}~dHd?}D~_@eYT&E^JH82mG<%T>h&NUL8cEzOS9%sW%$tVZ zkK=5(U;Cf7wt%M3)H}Yfo2<@J77zQ{;r+fR3(J(s;1tO?HW5wc^YW}OX`^T7ok#U9 zlrfa)ZACCeH3leTcl^1-E2uSI)Hc!s@qmbbA=J5mSK}Sva4&XBgW!7HDK$C<9 zHX@B!$%$h!_8ZF?>XhIuvkDA(C!i}M{)ItT`juE&@=&-sGV4;laXc6vF%k zqZ&Q41`xyJlGn_P1mm1eZ6ey`;_<*_++<;wFU?!0^&}wWzXCSqKcj;Q~-SzV2>2oo_Z%c8fvp)=99xs_0ym9ko8rKP(i z)m|Hll?aR3$!NIDcEL2}Af9=_vPCPY5e#duPeP{U+RF0LLZ^AFnQ9r(dn8N!xjhj} z<0?A-_iaQ5Ajm5anXJglsLlJ|d-qao>mF_!fwglQFTLn@m;|HGS*jYTJU`gq-180j zX`Al*)lsZ6QKKx^ajKoD2T(klyC=|Gk)8B7fgVD=#xN8YF8{%mRKg}2Y}V85Iw|6z zLDREoc9UADDY@>-&?8gH4Lo4|g$eCa{oY|kH@MdwPZ11Ng`DNMhEGr!sS#79`IPnG zPt7z?(?9(S_obP;ASHJqh;3#)W(JZH zmYo8W{V?=fXLze(*?J$2Gpo zFK}&T6d~)C+h@;a{s~h&Tyx-_&Z1$bWhPnA$kk0Oz_(9HoVeWzo_!!2qvqycN%v!wSU%@KGhkuusGi#4i7fB^0i6OHC`x!9e@G$sjL!YhX&|zHkM%;`4Th` z!(yu8i){aJ)8FPd;YZOlQrpE&#ips6cM;hufUpHwk@$ToI$2%7WNue-=+si=GEr< z7|hN>NvE8IHpNWrje~hrlgdn^Cp=EU^lmEf2ckhHp-0+w-8r0~q;ptcN-*fkokYqK z3P6;pZn2Yc$R2RSBjqMhasMlMEeH9vaUOh0vDx6$)DuTT;uL{!rBrb#?1$qbz1{Z; zT%~TDsWRbNSR_WbG>^5fw>EI9r*T!r&0Hnup;QBr~oM5+Hw3j$~vz2iuhRFV%lmiKMie7udq5 zMpA(D60%o`G+Hk3XR%T>Mr1n={Wn&O98z0M83NOJN!>Jm%YmWPaV7+fsMqjCV>b-Bb@P3PLlu{PM5vJ>6HjhcHopus&UZL4RA<&)M_AieXfg zn7fUQYYMRoPg7oO#aXC@m7!S<3wu5`BX6_8ZDVFret&c~rS4U}ps-391S_JK4Y=Ub zjKuARhdCK&Jz;-iQ27sH@#q}32~+?sWr+4GGSFp)LUV;eLE>~m^jqwHZ=tq1t4?($ zCnKQpZs?Smgt(yvd$8?LXGMMJ$6=<1Le$M0_PVyyqy}qu^M@-bgG$Hgd2jR{djLi^1gHOAK}O13W}Eht(f6e-WyS&fn6Q+n<~F>(d?M z_iJu=*uc)jp~s7$qNh=aBOt?#F{T_58yRFN1QwI)l2^)O5Vgl@%1I$zw`!{TaV)1i zT|>#idTz=vzOC?os9=JCdasYknqgv+74AcvB%obi+gVX7bp@bEpesW@Q+Eb&C4pbs z%@2-T+L?U~uLBf%b^QUinl4K(KkHf?V=i>2kxp-Q2%N?fLOoNg@Q}Mejsf_b>9tI; zFoR3^j}xe?WMUa3kD6}GMoE$_B)*@pPDE!|+vT&rn8|O)lNuzYs)Lc6L3Vx$((XTc zf^;!W*#0BZgU}KXRfeRaFvF49;Jd-DVv1o;rFb-!n7EZ1M)=n;;;V>g<9MN&SP?@7 zES6Z?DtvImt`-}3Oj&A8?TLyl-nPJMqRBFI%CnGshq_r|>jUU!%1A)*z9VkW@0sYV z-8H+_jppFpeARG^Z312O$NTvnENj-u?bENIu04exD zR6m{&y388n*%u{$G&R8P4qTPAj!^add~C})(yMxx$7PkNIxg(or_q7pe_Wk|)I?{= z;BiQyZ91=vrh!DRh9oLOnaB`@h8&TczJD1)Jsr3yzkryaHdAYi2 z=k6u@#y@Loo=2Ld-f4#BVcd2{yZ37S(o?*OU>0EnZg9?lG&Xff7ioXx0+}lK6gLrx z&@8braz2K;Ym&E^oJ>pZRg~$hwF-kv0&d}qZ)<57;NN#2BHnGws;ZO!J ziD1~=P&b!_IvSwdcobvS}Hvq{&}NKRiVTES4Wqx;WqDg9gPActstsc zv2pm7O4l~=@ODfRX)=EY@bF;64>D5LGLpU;|Kkx&$Yn%6jR&CUX$<7`}N+t4Inu@+ApLsdN0tcMq^0h^_O+F*mlZiMVD73m1 z+vvN+D?a)ie_#@(M#lD*J?{Tc3?z-}r9O=uqf}Bi|2-(Joj<~M^n8Jytl~$C#rOOF z0|R@Nt#P&*w`dk7@y#(M5q0XMD^Y4~LLPMm6-K%k>Tn>^>|H-EpmaNJ6YqG^|ETAH{DFJF zaT;MR(|>bPJeU^@(#i=m{DTbs{d1~07uii99f{JRyOmU~=7S!!M$Z0(`Ts!zb1x`u zh6NZLlB48eB$JowQ_}S9dxZDuBCPfp*B=#gQgccUb@K}c7Y7e%ffSP=@t5D$w)|3^ zz)5wFHXzpgtgTplxzkr7Dx^w`?I7&&C!kVsDFbyKS?_cyJ;b9OMdpU+ZBQ5s8{GmHeB$#ZYP+;OCf!p_QMNQk@gSLKyZHO-M{iito6& zz~oq2VGW{KddW*0AhicX4^7{uA(S+xl5KBwV?4si=XRo`1BeABuU@}z z^$zfYK|7?t(sbTA3GJ7Ei^&moAJ~csw-I)6Wc8dPmh^$uP`-tmtUcn{X-T zW9Be_n$oD(mv(lgOPRp1>|Kpql<0J3k1g00$5V20=O+9opb>E|h_{wKYojck472_t z2f&tkU%3mY%;BMnG2mD4NsU{7txb4M>Qd6Uw6^HmR1B}R$KFq;^Bf)D$Y|~@w7b0d zRe8M8B=)M-a{qxuyMTJmFX`?FfD7?%&2ie(Vx8oZ>b+v%X0Kc*diS^h=`N@?I#8e~ z--nogQKThRB@BhR4xA(@j+*a7LrL0n7pv%}K^SIBL49?J!w)cx+%Pid%~sWd*4+kM7pM+w8z zSl>p%FJlvwv2jaQlf2bE!<-E6s9%~BADPzT{hsoWUc~s^RuvedwRh)b;(t9L`2YJR zI9->Mh=g`>`?!5x8g*wN;-WTJBf!-GJD5@0F0}m)ne#!U`aj@ccAV!%mZrJ(tta0k za}t()yzf{(m1z}5?=tVTB#WUg76moT~wl}uQNSDvRy{$jOp zB{Ha9t((i|rUN$>y{QwQ@`|Tfn@&($ z<5c>YFr@QhC{2cn|Zt`pP>To}=5~ zC0dmPyX~sdkIBgH*L*R=OcK*cKnjI#uHJD@yjMNtT6k)3%46y~fZ&;v;98D*=yP7T zowOEH-7+AB5+Ik-C?0$*whYD9Wbe4a0N zEHO(i6p*-@QYV$u7U0Ao={tdB#4F@ zDuEd|#^F<{V5K*@!h^fzY33*BwlVu9Z1)IB+~Em=wC_La)|{0;u(_JU!j%_qcY~@D zE&Uz8(|TKgo*%L?8^yzvTZ=gU_<}3~vt8u07_daI#Q~=3YP&f^sa= zM*Gae0Tx{LRlD2+lks9gCI2ogfh8TTZk3;(4*B8rf~wczWCYjGxGTVKm+3hB>su92Z_IVCz!o1E&m4^ zeE9WaCer+U!}K+XyXT5~;~4S^H;epy$0i{QAG>GT-RtL^ZTC>HndN&&B^b`{a5p%V zbRLH!IPCihj}L>zJzoNJ@#XD*Kz(jK( z#ZCB&Je;PZfc7kB*UMxzJbCX3-&(K^^3Ry5+Pe&D5~`2nL~(;bBL}9brP(U}+K|;7 z$%FSW`w(987O7Jvt#ymLP)SE}j1Y?H9BozW{3(hu7Dm~eG*$L7D;sfmPS_cb-ni%a zlak&f`;T}lfo;ZW+e5leIr?Nn%2X?T5N;h4Hl&u^5+GM2=sIGD07#w35dLE2uTC5ediSZ~ac5SP{PJ&%< z-4(V-n!@!0AVeH&464ifoC)bWIf)_YZMlGuLV+Fu(^fRYkLjdq)Y;z?w9@@rc2k}k zy1~29oK-a{BI5g%?q~&j`QO(Mx!*Kzx#ZS(1R6-Nygxn0a`u>oCCFd-sW0JpE36RP z3JaLsLK84i^KsN)e+xKmG(UMz7p8wV2{C>=(b>RN)QH?`t&r*97~aeW!|d&ibpT8d6{lQe5dxW5!2 zPGV$oAO%@QET5Nts+YNe?=%0&rrO<>PFHP4Q;?jUL?Z!PnaDbJ5kb~UQ)42m%1l&} zJ3FS)O4`nKc5aF3a9p^7w}tQc&DX7rs-&39`?+{006*TBp!%Z%ksA299gSXYr|;-H zBZO^;9u=*2dN<)WC2d2LQKmx@#mAbnTS@o2E?M7%#|6_ld@bMd=348#uI(U(V(B29 zm(-uY^N68n{KHAxIuZ^(a_OzP>^bt~V=v$N9+i_DEEZZUD7&6(UW#OQ-E&i_8*XVa z2NOYs511PG(>C;nIGs@t8Dl+S%|Bhbzd5`()_1QZ6-OSqV4d4rTvM@Y<6l$>GBGaW zs?7{Y-TJqEUBg~;s}o@AM{7@cHCQ}xd1nFtS)6EzOaHFS#Bw-ongKY9ue~46sm?Z` z;bd!Wu7{c?N# ztO6`k#i7Fk^R>)_*g^1A`dJ|Kg;H@m)xugA!3V=dxQ5h56E4;TcvXvTv)mmR`dY76 z@Av|o3BY;N7K0K??j4_GPA^7N&Rvjo18G@0Z3-u?DCLxrDG@yh=VB;c=B|_*x1N8r zRKf8=MfU?W{r@}kw{w(J(e?62DDQZtqW3+^JX|dAcrFA)$>bj{-mv`F`+X6VD($osyLdiV4oTJN`V6=UI2__YF8FEaUe)4~<0roB*Uhr?gUMD@yMk{3#-0p>3 zmB{hG&aEh_?OGjqi8>wrN_S=dhQmV!)WS3SgoCxL&TBwJJjytre}VZ-iXeMHg#bhD zP5D%;R4ZC*U$zi)P;DC6$D>Dwcr>ZL2KN(`W^(4Tak|IK1_8=VhIumF*T%N}SH^* zgzn8UZGu!6wSf+@)2)$iDUi3GRnOE$ewGk9QxYlSNSpD#UfZNqZ)?ya5c199>;*z4 zHgQfi1CAbyME2atMEyj!48z<=+a5x<^?`qm)Bmi_q$G08J~VroFaCbX5%}N?uX}dK z?2MVs2e(RF98Icv4W~7YWogt}QK-VsPTWhx)0|yIm7;yAY^W{TB3U-)ifQPE6Z$`E zK#+~MGS!&425Q*vyqmu%!B{yi9+)R_`wmI7jft}j38O32YF_Fg^z*hbw)f4;^K-SZ zfS>^I|9NniAUL}xFu?tN`3HE5-@_vy!274$T>0$y!;mHP%)pMrT}1aOwS7f>zestcPi^6H)kCLhpyL#n;2)mbBZH`rXy zv?%zKd@lcGy1iDdUfOdn=eC2-64?RQ5Ixn!*_nty-W6lEk=|NlyWR*wMW|}5aMo*W zQBn%~PaDuUJoSthP%`HvX2QgRqS@=m8zz_r8MAfEV9MB12<#POn+!tI6mDfte2z$O zPFfxoWOm)F5`Uu5c1pg(DbQrSPejop+26`EbL|A&TY)X>Cz-c94t`&cp#YF2{-m|C zi`N7%U8mVC3-)bjG=aigqZ<*+07TzPZ`(e$>m%Ic)Opm`Xi)UqP=EA`>`LS`L$rJu zC#}n^TGry&Q5ZQZPp4l0Eoj15CY+9yvZ7dH8%>ePJVzTnv__+I_p@PWviDZ<5rd&7 zAf(`xYl%y@RWV=EFtmls=`lU(V>)&`o#<~0j-(1IDC|Go+yLbztr|fq)Z)0?3bl9l zvHK*YFroKwKj#d)~={v z`#_lQPkL5u);}+*g&jnN6y{1X^9r=5TboD#J+2GD4+$=Z%7?!0QczRc=xwwWI z&yrtp4mxWguDFzQtnJW<(^SdU zveT6<+}5r}?47@(UnS`J?jVu14Cyx$5S+@9_k)DQbGU>EB`M1sr2?vhta?^F7;Mji zMC^|QX#7N2$Xg^XaB=zg8^hD+7FG2)IJmRXm~H9rDeSEF6|Ut4##O=B);uTdJ^rd+ zw@O=pBa8M_pSOhnsgb5)F{5WAVTFUaoW+B2D@8l`Qz!#1p;dMkbbqr574JQ>$g&J*{=5e4Cw;Ag~azAMz*J9{2KR z&|fGIZ+N#XYWX=lHfb*YV7#A~@7LwQ03%=CpqI~EVpnQLw{k{%Q_|}5HbK@jZ6#Yc<-rx^oX1r~SJ;INnFcx*gozvn=*whP;?tv;WHzkr_;$+T z&T+V(>1aj%tJwa4Nn$2yO~sY74p>y33@JdPW^|{|EgH>6`zykgHkX<`@4RNUA9Osi zsi?j~+utaYnIP;_8p&Wk&c5lLJHMlZlKKse&zm|O%dM}JSem3 z=jb(B&hWe=HI-A%AiW8@C(sSFrhZO8)7O|(Q86Nfo+BWq86*+wv9_q-tCovM2T&B5 zOk#f#HZMDYnYrBM*{;S^{hOJOp{80*L2?jm8~w9@ac6Lq1wuBS_N`Es5&PD#G>DZ) z#LUk}jcOx@uBv-R4)ZLm(*rg?S!6bmrdEuoTrL|4Gv--MtA@PSh}J#>>hQE?fI_-z z3G?1X2~vm_45|07u_%*{Ju(pf9Aru(4tl`%wm3A0L56-uDL67|F-+Pk9liF`nheMIBcH7*QAg+59=^C94*y`;qrMz&V|4SxNpyZbPYmg0b2@X!Gz9a zo^y?I>`%`)Hrs_!cJT~7Wp~z}Y7x_ufq}OQ1PLOgk_AH9cVei}&re7)%V@MvIA*(a z32(@>Ef(ss}IEoyAn)zCCwPd*1)56m6muTp%tgca!g~sP9=Pi3<+SyO-;A4aHoE62w46-Z9X#;@ zpe@to?CcNloWYuHeFsJt6{IqH&@jt_n zmTQ@t#|azSYog*DDi@ zPZjK+o*r^?BDvX2GiH!md_O{T=Tq5p=Iir8o@KDfCCP3awP~j z82PslX7~>2H{?xR9DI02@3`R>PgC2JxW$nyacV`vE-<6?$Oj-afhLZ$fu+*P?lD*x zK@A>2!byU?z7 z6@%AGDdGJzz48%v(s2P z(qoC;qA+YmiRK&AU3^V&y()nzDB+ysocgqj8>A=+>5^lKHZHuatD0EU zHa2|itD27ZR8sLg%BXqt=2Xqf`@>aIb1NrjXIa!XGOMm(`Tk4ee9NeLR8n()+{>tW z{+}Dg{(?$s?hfF8yLgsS^MK6REA(VmUAz2mU{=-ae~bS+m3tQ}##-8w;jbR7Y6RSm zJ+uBqkv2WVYlKn6{H1(@aPXhD8xi9V(0R#^HT*xD-s77ox&uixnSTu7g!KD=t9Gfr zv-*b_v2ykbF~!f9ych1UuQQM9Mqqi?q{vqoO+n;Eo63Y4%Y7W-4$NWxf+m7B!?Q`F zVpPWCgqPo)LXRyFKg{gW-7 zN3;lBWznUTl0!F>iZY+ppIm8eT!! z?P)}(U5i}fDQ;8Y4IJinz34&n0iGtJ=7jFB*Y&!=Ab9&*D zS6+zaTGHMTA6x3Im%@&|qYVRxa0L*P&;O1x-v}aKOMi^jt$aIl)tYSDd5giW>#G}i zK~(iys+Fy)Sz8UIc9)|(#odxIO+aXen7Q>Rj)NYO^;ZIW+`0h_xq)O``u;pm2^=sH zHIe#zK|fSnf?^@xt2!He(NgHq|qrb ziWIFb!yh1Xw`{{HE4vL|-1_!!kJn6Y(6|koM9;PMIjE<_&Jd>1*4UR$|oyd4a@8B)p1%z$fC3N?+SEWgMoKxRnYCI)ozyA5b0cn;n zQcNTdF2*O>WipT354kP$F_Bf2u?{gVh`}ZUqe0UMOZA7Kfel^wGi)R4hCm=M%Qk5q z3$Bk1d%7+HCCv*2goW*1HfLrf5F`@KSdAyFt&8i&3WM)l(Qo~CnSbe&;id3zJ9wJc zuLKr1<0I)XR-2yXUIQJ`H#sZzA2)$;9mCSw&e2*%YeT5yVT^EZt~t(iWpo#`8kz>S z(;J2F6m62P!8^krc-W2xR?STyARtmO?$d42t}6aPZLpB-MYzJ!8c}Wu7$Ms_&+e5A zcjbnzG~)oP?oDz`<4(f4)49AK>>4z$=27jvl^V3U8U2rw1|Ny%8#+5(gsOC&28Dmg zN`JIes%RMRBnXkQDi^2t_-=*g<1p7@3U{94T zrRW`OOqDE$M&KAq$1~?ahYT^Wp01XD+Kcbx1p?wZQ{F-=Yu&z5okH8clssa^z%vr^ z$lOdAuRwSgVke%7>sglLao`Y}mH zr%@xbZd*>B+w4@9l!U&TrhcTzme?iq@IyeVvoZWa%|eCH{RE7@v81PyhA(X(7?DSU=)ARt>HbQJWmFag|eN zQ4$ny9gD{gmIonQeOF~=U}mQf=NII(4)Cz}>tmwIONHvm|NDsWRGwiqY1duy3^5y! z=(`)R@Y3gOB-b{hYv_`}L4^0TUv{2{vMBbjPzE_f@X9N3-d0CZX~Mh9|FKO06@vs6 ziIIV=NR9pMRezOnWhk{`iI|x(8swH$W|5T!!NIa#{chWc_OjCl$b@|F}|Pnt46Z)+9#m-aRF{q6B0r4)5l9F#YVGdH1-2 z^&Wl+Ns8Ms#N6q%6dT>cf?xvPS8h~hqjaO+#RB<#{_ly7&K*KPr20EUs_|r!4t1Ia zM{x}Ye<`a{Lmx>};}v&7BIo;|s-paky5M=8&=N-+L>65-aML9Sta)XB6YKfBpkdEV z@uebFR-@D+htoA@|E=6!aw00mE&_sG1*7;n$r78fP#eSY9qRXA3U24RFT-@~J<;s= zkCqZ8)5BP~>i|EO*DFMlvyx>Qw@cT7Ag<2-C)oQD{TA+_FBb*@rralr)Mik6v^Fvi zk+RIipOT7$`He8r-UigL2~RR1xz7TgNof+=D`aSC_90tk<<>AmU#+Csp<14UpXad0 zX-aq7n|CTMYST*$;x+=8Mhl4`MX{8WfRC^SlKC?e67W?bub}xF&2B=z;(u-lHo|S= z&F@cV?D&TpU+?K|k8XcH4MFrRsy3P?=`;Mc^c(1rX5!CrQaE{D7v}Ymn9VR~LCh@N zi#aPevT_kNbs*5<r6yNxCf)3djTk0RAg8;QtJSnt&9=80gPQdW)>ccqc4rDU82ZlSfT()>PPlEBfQ z^C2juT%FL$AJ$C(AO^G!!JTu{eRHMz+G&#j)r9&DS<$t)AiCaAFd~`!qF`GBiu#t- zgQPQA%PfpH9hg4kMK$duw#q^%76yL9MQ%yE*rNyXx`Nb87{r6FRQnTe3$u%xEqLwRm9A-h(<`9?Ug9`t3U5bBxz=E$~{G9g5P<>wRgNGZBoj zRknvu{rw)m)0zJWrhtGTu4G33>9K6Meae#blm;W-*YJ))sA^p!P4)H_V64_MrG;!@#3Ec0@ZUsiS$tI1#KpN}T z(IrUDHJTJXTDQmPopU?Y~4sYnU^hs^U*q>KKy@u64%3IPq;%{9>i4F03`%1OycYEUUV0w)%ZR?^fhX_7lkOfU& z9`h1gE*7LKe#Wbp8guUHg9Vwc z=Qh?|v}ai>n~s3t8Kx_p*q!>YdZ)X6Hv{lSXu`+`rZ$p&1-`OCVimJO?m;ZK8TM82 zHKhrGYULmM3T1|n9`$-o`%{bx~o%>xS zCuL!?VcCWtxHAqp{^~W`1yVvK!wFBC?PJVz9XC#yk7hC*yW-n7^3c_&G=S=;^6SX< zh#kj6;B@QNVw&VJz2Ej0ARUw*`!t+8WKHL*ZmVJDH&&V{G_)@Ya|YlU^(NL|Ugb#I z>#TslqxEd#u0h3Y{)HiJQ9|0C-H8G+t#1-%&3E-*6(mnTCF@9Q$9l8tc&^RKkZ0#} zn!#T!PoWkd81?y(nF%eui=MabOgLDld@UjOuVtvhO#LaVR&wdnns?w?(tzm~_1cz5 zX>E4K5NmODSSHn%OcDp2yo`!gmbug5#6nSC#1Ri{Yj|5iL>1QM)(f+elY^{erZu<4_o$P{=SzhU3$snHax&>nh$`R{?8;A=u|#0U*+ceF4iFw1#qde8eJi~DMfO3RPyVJMQRbk^umVG4WR1G zl_h@4W%v*p_%{Ul5Pkrvrz-H4HTJt_+@H5!rfV$rgXSXeWC z7EqR(=w6fat5AeK`;OZ5B;c5iqPHa4eQs%2lPIXO}{_h@I}h7R^>Z1W#k4naaQ%iw`x3FgRN%12NA|X zs-zj<%j;7`#OO7Z-RQA{d5%H<95W37!%bHu2tR<$b?kDQprN8XK@+&{g_8A>Rp)b# z?7qg~Y=>?Qiaf#1G{0Y#*S+_c-#R_&*%_3GCj`d)Fc>Cgbt!cASO%4fxv$jc{D`E@ zW@wfh5O;alQarl0vEckk8t1(eVQvs??a;&i>w92}>#7Da9N#)Bclo!(PWJ!#tOJxe z#!KvCZub5w@UOth?nFVC@!I#pi3A`fwmiJ94R!vSr^{-NtqUEi(M_8qnaW_D$4gzE zKp}}g)jg?=E4ynzwC!$T9H-HRQCuL>q%3T6w_2S$NBlve(!yecxwpB>GGkxtv?hgU zw{dK^!otKfRC0oPabl@b-Lv>oYrRVz@e!<0_95%Jhs%5w2YGE_z_6*AJ8+PRy?GH5 zY!WQ4ahukR`M`eWe`eCx@ao(8Z0MFx=4ZSBN#@@dNammal>Gq8d=6~<&VCM*B3RGo z-`ANP^TKb4P_17&I8fFpUm_AW#kq39bYOb^BoXJr+oTVJFM4z0Zqb9)gAZ<$52wQr zftYBN568z4fjD834~NGPaW@#_#B92BMw7mjRtvPsd;P74GQMmxH;oCdUYQOr6TtwJ zQT%nti(1XGj)Zg1^BeJOE_Lr&VYiug?e>2iZlbMy)=v8>7dpOt^%Qs!Ac``&(LPiD zd>zJtxzH>od)^)38Hk(v<@AR(;Hcz19pRS3Y5?_|9T$|kp&8ZSFwoCUs0LHxWy)py zO)xK*tCx@2b{6xml6K3|cdf8>Y&Op;F-@Qq{jA%I>l9H5M$4x|1h0CQA9n4lHhKV0 zdfqMt;io?5BjNpwycEIA>( z@OO5Khmya&b_ul*02HVG`kpvlJ^(gA$-m9Fuyo^ar+tGXTRVu59^#hZqx~pS)kZ9g zD5z1kej^89U*3P&o&5ax9nz^O6;*<`Or$*1{=7_!^TsWvX6XZ*R&hbullnG^Z=5|} zR_CYqZD2oV@G+Smqc?Z*>Oqn|=iI@N#PrkdPP&&^@oJHO zCq3Gxyj-|-6EXFDy3RrfMwI`;>xq8GEh2;2q`FEkPBRn3eEzkTE!=l=c7)?QUJ-MF zlr0!Y$6TWhTllXbmpQ8Ssnz6kGFA?vHMBsskDM~;ptJ80Mn|U>GcNtuIGo+Y{5DmR zO&-~GMkJ9krRQATh}d&y^n0BC?Baf`QiA|ja~6KV=$9tg9IBe%g&88j@Jl{fSz{Q$ zImnIS{Vqdk?(xQVdBfLV1M2H?2!<9Mm1%K{3QAnF1asds>LNh8Xt!&x7^H!CN<6h; zP>`jZ=@GrY!93d-F3=T>0Sp&e4C3l8ok zbgXQP8((EcKbyobR4a9~{#G}KYuqLa4UVG7{sFgCe=D5&;dOB#2YEIP@LT_q13%ww z!=C`sqf!rZyFu@F%)gG^;|xOSZ=4&kC-+-oE864jhk^;+*Y8YOqh$^`h`ysT{p=&` zqLPe-96D57TW^N}XMFm`rzx(Q7aqI{~P#yBX^Xb)5w>)ukt*_s`EyS+`M!Yz2!|% zpSu@0Rb{<<$0wus5&ZrA`1s|y4C{Lb-_w#We7*E^pxq<_aGSB1oZQ{AmR$y*E?L*V z$xK%VG%#!>bN5PG2(wVBpW~Uk6ezpW98Ma;&rR%M7Pc>w4KE+h2owmS1w_3qK9fMU z8gC?#F5?57-!j@*QQqDh`G1E4&;-UyaHF`)lSOTF+iMlRh!H7`B2a;*(?uJdD4hq` zZiESLj%ny*x-MN!#~`K45Eg_BoHx2@7y(_!QcFA$bEteTL|FiL&{J?UX5tS47qqN{ z9H^Pe?1QadXH9EQnY4z~&{=lhVjTR;@rkTXq3h&5brUO#pah0FWxd5X&mIfe;;HYea z`!0S2&B6S%BZFBkYf!g_z1Xxs7v9XyNUd&R_N z4eteKl(l_I*Rtm9+qgGi5WqLbJ3N1oM>k7JT+-yb>OkeM`zP(CY7`O5t5 zkeA}WW7^l65Z#(HXMd&35W^z*-~?14gMIl?c2QSEZ3fnhL8C)Wg-sEYe@!0!Wq;!R zv3?;cS$O9xrNSE#{%ZM4eKpxPHoL{wwb>>c;?qyt%QiEISQ-Oc=f1=Eg75ixu#Art zk>gq+{XGNV>mNX9)UG>EnLU(a3$j}jj~H=oz0NE8Nr}P)eSfsYa5&)WA5akN{=e9^ z#~x1rE?MAh+s3qQ+qSJ~o72X>ZQHhO+qP|6H_y#xvzzRz^A%3&S9MkO*h;vQrl!N3 zT@`rI^9-KuWoW_v=kV;08EMoGa#%oQgkR<|i>Cpqh0;XlbYDc1`eFBR&kRm`^8zF`&#U(8Hf<{ycHs(DrmEY#i0(kDp zQ_QKF-E}LeKrVHRxq9Dv+}xk!Hlaf>mdU<^S>P-e5ezaOot z$ut?$#_}jrM?9&dnB0;oasVdB!`)xkq6No-w)CH0;drwyy~N*uc7+zq%D)|3aY?2w ziqcV2N}ZoTY+;AB(XG!EoA{4bDvx*=Mh-8IcxbO|@gR&BcAC(3Ch92-zin|$EPerA z2U*FOCV1}lt-V+HH+8^cHe7Dtk&5`4{WfS3@Tc5%&=$^cAD+|w}ieeZCIW1H<6y% zZEsU6&?P6`Z-Q)IgU$r@Xm3E)%INMh*u6q^O$@oe>vg*DY5Y_Wt6;%$U7qId&)lO- z2Mvy5X^=27DNz@S{NU(R!+8AAAv?TF^8{#N4y>uX<_#Cwvn|iSyTSO@ zVs_2P@}uq8Ad|$sH4=R8L2!-UKBEtR3cv3KY;$j{p}3TN>!6&`h^R~0c)npj5du2P zY6NoI_h50YRYX99BMH1!8RSHDhF^OJoYC zyh^yaf7D?gRAw?9mK+UqXqxV&tYuuseDUI z@HUAd28l78cZ)I1ks3++uz(c3EcxuyZ!F$2jGez@12}IDN5L6L$?e>!fb{@da|wY^ zUalYC!=;O<39Q+{;~lT*vMb%TdA%#z-J6j^O zUqi4cvey=0ScVBcXYHdnhKC6jmE3nv zRco_Rua2oC?Puaw>G6y?bp)SqXKxQc^fa+=8GXXr(sIxuT;cL5u_GGGS@CcRTg*Od z0gTX9iGAGLP>Frga)>0m4U`SMqmpXp%gzwgoaC7c1t$sap^tQW$(z}VO zMcUnNg=sJ&$kfK^YdnDCDevtnHFN`X##0Pl#kBUcg{0W*B=j1mMN08$Ont>0eW%$; z8tMPp57gWbL+v7)q;>fuw%!^&a(H}tZh96Gp}TmvVz%4nta$rb4-(GVLo9ZuNUusZ zeIB~^$7{r6o20TU%@VL;0!Q@WR9bS4~Y zH1kJ?rGE*8!)o=D8{EHVBg|R??6E3;x9x}oU0dUpSa1uk&@sRKS-5re7JHL| z)8i49?DbgG!SvtBzofM%=4fgMYOBh%Po3VhGvj+zntgG-VddrJMdXCvpIZA@{k4aP z0(6HY1N4WMK)b`*pi@d$ zXg@9-{jhQA_K#Pb;yWqaBQ1Zn$W2`L0F9g3Bh31OeNC)H4WM*2I`q#J96(!-sYT0U zsdn6pp16n!&{Bg0+&VaDOcr=LV7hiS0)VU@8 zoQqP;_U8_ae&H}zPoQ{u&R#!dtwg-DTfF8wrcK(}H%kfOJEkWjHwAGn>YVAaCC~%` zMB6I?Bc&?{Unm$B$*ExGMq^LYUT(kuQPpN|=Nt#`wE6hDJm2pf_AlSJG^}C4@UQc` zo=1aD+V@`UrRtz+9I~`A5WQxRTSqd0s;K`(Q9#C5Z@!n~4NYUeDbtVN%DVe=NJ|zO z{f*8ruc4GfHIG6 zX3jCe*)%2yb;~JkwZ;s?E-80r_+R986=H=|i6}|If{-7U+}HlOw|K{qVg( zte@NO+xGqa{5;}r*Z0T6*K_RG+bN-FvcX}mFaJkC_YYIT*!{i1C)mS^i*vqtL)Jc& zK9b(c-hH!AgZ-U;zW}`^F8*Yw;zXphSF6gd^566>Hb|Qyjn?b>vGP0`_DbALHn<0= z)?YRUs4X?_>t+MzS|10H=@q+1ECZg->QtQ%l!X1vxF0>0(}=MxN}m=^$H9_M7xtR$ zQ5e~(7g_&n*vqg+9ui-tY}+n4=O~=E<8F0px@gjY$3LLiC@(Kl%>( z8VzHCFaxcnilGa2fO_Z0drwHq6T}`|(t(x5@z41Iz0Wd02RlZv7z^XPlX;VL9y_LG z(oCQ)V@_ck)N{3^g57>)ZUUUf{ET$eD_3&A@FV}T%jB5D&{D zp2f+w(b|sDrPMl;H`nhy^j$@hx!eapZ+pw5nCHp z09N%$-^FQAB--P=qd;GI|2+T|mLkgR+n4*ZB^pzNx*RREgNYNjn&FRn=XiVD$o=;$J=2mI98zo>EA5Bl(gnVrbZ#V%52I=#I$j4gBkrLbs@XtL`rN&28Z@ez@YNiDC$8gwY#Jk~Tkb3>ixKC39!1!F z4mRZB0Hv39EMjhx5k#JF#;_8&+!}aYXU$hDel24c;*3*NAKRtV{yZs_hGL`(3q>hN zB0M+wWWXQ8PI`*CIk?6Ax@0acvVw_*oYu~jzAYiedZpE0oyEYki~4};xs^+95nA#g zpHEcTAE!HX9g}(;I`aZQ6_KdKSAwCo+#avxR6ZQSCm-%P{42NbNc}}%Ewo`EQx$Lp zD7LCmaY<-n@-*&0{_FX+P`gNh%!Hd!hdKX@j{yF%a~Uz0C9-fcu$l)K?#LYI?8c;q zi#~OVCuVdVpU!7E`ru4g9+LlF zWG~&l1X4hDV(_uC2QQbnW3j^#tE7P)hj?HBoGe#{6uD8w;p&_X*~4tX#(9F(C7D3! zIcZ^yH|ah&t4rYhPQFS)O1qZfuRumz_8E|Jp{T;Cq$*J&C#dz5r`3TcwPq&JNrLyC zMG7M%)1uM#2Hx}bWy5^cSt?7veGi1o{2YkkicMm}Nf2k&ZuElWNCajrjog0zCwDYQ zM5@^%d;P*zwtW1zcQ6G-7XTRHjBbdjQ+ZmCq1*LjhN}=Po$V27)=&0|T!lR;Gg4=V zgitj$xs#sxz|Ykn?>76rPXX8Y_-$^txYxY$125ohlI_=fk*oSpr)xMASSJ>bY|-2qb*)%47a_ap%F%>q%-bI8->MU0 zbnn_=Vz10ao@_EpJ(@mO)phQazm!+q>({{((?Wg__Qq)$L6&^N zQ8lx(K2Y%ft+EWxe@MnH@V$OaL1>?$dwlqR{ISj7-3=`nn#gE3ap`(UA%pU+78|#AEX|Z2C?SH?3$m^E+`+wChgbGRlgPI$*9L% zEoGh!#+*`_92Tu$<#=2}>R}WrWzNFZBx_~p>BbM24j1bJk5*h=b^L-;^UMd$PiE)TiH=EwO= zeGtKn?=NAM=@d1${-)A=DP=L3^ciM1P%Hi) z`LY!<10Wz_4nw0qkGe1nV^U1j?PV)WGM2}F1 zLM&~@bi5KFbKw|%wB*@cLNkIuDHJhdL4G=nCT? zvQd1j(FuN}YjmQa%YvSEqnmae>=!W`Au(QIKLnc$n(59#So#iL1iOBmvdDb}UD?U4 zjq5fmD0&}g1ok8O9Ku!!lS^X7tSmO7KP5(DM-2*bn7BMW?en|ViFVP9W=6U;9Rj3V z{vJvd??54#8b>LSsyi>heZl*UB4Z*iE?q(JZuBfUPnxY8D4L_@Y()~byU8+JZQm8{ zyQ(}YHQOw(ElI`igpEqbuLTSX6M(wxNtc_#yMAt&P%8vz&gw>}RF&NpajiMIWE$T6 z4&SC`nHZshP>9v15_`*FQk(X_&_FqL#?wyf3su`xYGl^J%dlR4xtgfn@{#(%%0(7w zQXc13Hqk&b2s@F8$``G=0SVqfw}e?{tuBQKA8-ujGMq#Nees_K{nNN>k9?}1@_^pc zP{UBqYjjgG`9iLqecF(FOUs_dU`Lns{sqpNT*X~2F)GxR*u?v4rC4ba!INq55ju{H z#8wv(i;^N-9?Z?x%dM#wAm?KMZIS-1?xY#^KL+X9x>bqAYD)=&yaIn_E29DBLWvC` zzKX%OkXNICp8)}s+Q902`2p;(H9YgqJmzV2AeORq79v9!DzoCN?W<&CGVMU%HOo_w z2H-3A`-iL=to)7Zq8=UDmaS0TL{)%&^UX@DdF_~r)87wQIGw*zTtp@`a+CU1Co7ci z3&=UEtJLviS_`uEsDnfP5^TV4e5%{U1{z3V{P3Vi<*)qk(>=GPc--(J7@a?F`fCBZ z)GLR8b-J}?j-988#M*xNE$+{@Q>PNDzg@($P7F^2)5EofyV~%&Ch;_D^H#evF?;5* zF7bRQ0xDPVn9JK9`_vLk(fuA#$F4*`V9C}svY~vHFr_bnxIA78B z6S4D)ivA>fAdX+>q(1|fIoSJ@$UGUPTCDZMNBI0j@8&QB+pEy$Zi30H^pY_ka2k2{ zD&76X3{0a9Rc_q~TVrkv=FvT!9_=;W-A``oF|doM?#Ybx8}8BQ*4U8WoxWbZ;n9`o zRo?~8(fqGfM{2iLK&PqO9Z6=&QH`#7lBG_li-0j-TJ+60xaVCZc zgD1j!K-#Z4#IPtgo|d9wH-G&+6US6KORdtDfT}MrNvnd*Kgh6-4^TP#J#w9c_R|Qw zE=Ol?!(ETpA1C*-8Z#<%+{`BPuy1ABDl&#|g?XVAZyDCkUHH=q$b>LUb1JeTo_S@nIkTh$N{l)f~EBI!LdQT#13{nN63 z)wz2U2%`f;4)5Jd$UfQ1}x>AMWZo}MMWNnQ?_{=O@wMWgxlc<`uxLhiy zE}aI5(O9F-+IC~B=CvUyk>1JeGZK28)c%e;0ev$0dM5)zA(lG7Y0XA;SNKt-YqY2| z#bLA;vMi$(39FEvIFJ7U_3R|HaGHu>YI|GQx^VP>g3kVY-sf3yHwvNlfevzAr{I{6 zz7q&IamXt3EWMgcI)Wr|Wf6hBV;W`kPlubeG%e&2*Uf$~_X!S!^ywAS-2wD73gzh1 z=M8QvWviXU3{px=G8V5245fELuyO9NxbTxX5AZJ}n2k;VsnHwjql zXf!Jz&uKd+8KbO_HDNRmS8DZ6J_HztuC;^``vW1&(CtXb#bQowr!h&{{wX5JjHmwG zw`L6YX<}44M&aXR452)OW{+8>5GM+xp67=ab0-A?eSBr?)RALUV|;3NOYArVy*qph z<626qq+>%)BprRtJWZ9JuG{4ChM}?Cp60t&3-IP7Hi~z1GBsYb8n(!4_4k2&-Nl^i zXz7QV4Ymg3CKp!$I?b?rOWc&AIU)8c8gjAtuWm}@SC{+3KGf1RboPyGF<0v5E!NtX zkCnopxvg_Pw`~HZ@;(IVZa4d98OQpUV2Fk%z8Q0lBVO#TEq zZZdM@4%>mHEQOC;B*TdXpI8SBpA&ImP?kbslmXl zCpwuX9^>!$jOG>1IoSo%4br*`FQboF$~r&YFtE$ZLk?)%d@Lz1PNmG<7IY5f;bvyR zdy1hF6{2JaZPR$r=*C{6ohS4gaEAd5hM(3X3tJ4skUz^Lqa+7hr{SEbsWVJ*L4+@} zvY;1{mwgI6x&5b^5SS`4UQl+JyrLo}iU$^PCs;bpcr}3hRd|puduGOqNq?&}+leb> z>17oZ1TVUERfy4*cuk?{yciL#WKO*7FuK9^by=KroTe{~jgv@a&l1e^HE$(^=%>~? zC;*k@&NJWz+NT|hnl=1B1(sGZ52Xqk^3K!gC5`-=>V^9`lOudaG1o21a2tE23P?#$ z(BU`_cjfcAJjWu0yXWgQOSi*K-#;F+me(<6D%vS7=q^nAI<6MzRSn;p^yG7zw-OPi zVLKDjXOUKp^3UAl20v=m-Q7em0wl@IdKYttlv?T?)Sowj38qoh>}dqsw};@4N9eKR zg1p|gyWa>63k-@fLxaa=POzS)L_=e+j0gDPN)I8)M-?+cttn7bp?>K~IR*_~j2Dw? z_WTEJ7aVR1y#3@$iN6lvkK2zpEACDy4|WzMHSNQ#KEeAKOWAV$ih?8fk+twp7W8wO zNEge_(PWF(HpH3tIq9K5HX$Mt1vT?Q||ypd1?&%3ZwYF>p)+t#~xXmeqGHN>bH z?eHvhK=O*M^M~TI^{1U9^Z{37Kd2<>)^TQ5lBVFnya59?Ayx>J+V%-DNNTt4`o}4o zQbxPjd6o*|+wD53<3I@vn1Tq0BzQAK4#XS^BB8}GrT!LN6y+7vkO5d;h$f+V&2Dc; zaXo5-UpDw${VcyzY~<0jb1ICf8*T3fdui@rri6a4k5xjrJRh8)zi$U`wW9)^eqT3l zZ)azr_B*~l9zP!vzF&`s1rxRR1_FfNefqyxQb%uZwVj-1HG)k?xUhn&>gp|K|;Ph!F@|`G09|{q?nF4+gJolF>kEQ01=6-33wST3UJtnPa zYA#@d!5JY*97M4AP`qEv+f1zA#K_P7IIDg2m;sV`*Kza^pUDcvQHW_O*1KDc_%0hi zYV?dK%y;unEtdc8b^-?FB+dq1+MZUQP?u@@OH&@uW0$tB8RxLuP<~V6+44X%DLSHQS%qgB*^i^b+=#`0JAOm2$hC=#WF2xd_%iuw)egksGxIE0C zc4#+=VC#%O9c&%|$o(iKXmlK|!LeC(S4gR9As*1pYTym&$@Gg#&L&oTyjRE-YNzNc5YL$i^XQe*x~69;Fr@~f3PqO zGRyfj-=D#%rP#Sd?reyf*=kymi_ZFr5+&{z$}+?O-ks%{ zx1r1vZp%-T2p_;~yJuIXOc4!AUy&# z8{HuqOipF6ajCFev!;~2tz}r~;oA$rJ7$KPhs->AWIJU!*V+I)+%F8{bpBu=-x73V zQ4yk*N%rz~70y-PN;^6A%OAB_;j!BGE%~Qt%9VU(lAu%*qP}FQdg=5&%igrHKbtqs@!SN4`ja5G+rK?zTC?s z7dzexL($TMX{BHt+k!8&u{v6fIY_oE1^zU0%s+> z*XQI2kRLHmKrdyg3|wbV1Tp9bM<}0O!mBc!{cqi8Qak~#XLSUpA6mZjU+cGyw$BzZ z-nB@Tz|&yPciGHGu%x+cD(Jn96hEWn+UT*bkA|X=%X=;5>__3n_o!{;$T=|fe#Nm0 z;ju<#x(iJv)+Bz?RLI9U8@f1i?|wt6rsHQlQ(dxLtq|T3IEi(SHjfiDoy7oKNJx@p z6(B|mRaK(`nNa8jh~kSNyXiUNR!4e>a;FW}LKb6hrgboX|C)kbvFo$)>@(Fn2f6$D zRR6wvQz5grtAEV)>u{igogN4u^u*+LpKPt0dfNhgBKvE8vlkrd9*l|6jBQ;D)#!)R z8){4wl+Tf@sLJS{-wm{&9banpv%$LdVZpCs@+YHr%5eHs5L_9Fg+%D zCe<~Fwv&cA$06I+I{?3FR|jD&em*nfZY#N*F_Fjl7_LgFo8Z$h!2ddQI{+0d^mWsi zibP+%+d_6z!TNQ@pAc#fB#bC61=bRv?jDAWQoODmQCbz0ZT78Z)b5j2ha9TL!g&OB zl`)pR9PO|eZi*`9Nc8Oc8IDjuLW1o0M=HVgD2q5K%RK{_G#hFt>)Lz^VvxeWtnqt@ zuV-6$ycngZwE zd}*j@KEM;%Uwl4F5cZ-rb4h_2gwH}?drJrvN6Zb&gAzl}BJ=_)$A0u*t`8S^+6LeK z39EO=T!h;$yx@V2010|){ULk-zO4Y_{IhuNo~BEY#db|_KANCR1Sw9R>V>F61Sx)K zy=ozq7!B2YnoK;}s4$_OKy0xRFYVEs_8$W?v~3U*y0y*Sf2s^&Y1y!`(uZe`%QG*w z#&WVOOwf!ZF{}&x%NS$Q+g`$6&>kK!Ob2M{P=U~2HIYTB^|Q&@hfoc+TR34!ZJZ>Q zY7>At1JS> zl{HPMoVKtkcG;bzq%hV_SD!7Y?THmpi0GZeE5s zC5o>JBgf8WHd#q5@jmq7C-2HXIz<&5nXKW4e_*wsJ3|)@^R-{$W{a>}CyQ%vd$0!* zE##~dLxxk$p6FFJCO&g!ZN?{E8w5-j?%) zn$oHbW?W+Urg0=o5XwO&aWZwjn?-rB>NpAX2juYwA;~xZ8Fj!ftGk2=q7@|)W1h9c zudq6Mm}eTeji_Jbq)IsD#*FHk<7jo88?i_D)T6XRw+Bc=XuJO3@|>cp5K{#-gK3NO zs&=#Rc8Q!hz^tJ^2wcx2Z*;hN_#W6`e)YAh>wQ%d9r9}A5y*f*S5F(Bt||>2kV>Oc zNHkTKmL|)yE-jO&yG7=bg{*9m;_AG<$krKpg|;AR2rM6w_RRQXoY&GFbkJ^qb~=OQ zOhc-wYoPFzvlcOkr&-Fe5iIUVyd~X@b>#W;N^(gRJiBZCM1tuRgkv-Wc}zsc3S{Tp zw}yJw;W%gmc2Cx=iAYorU8{Q*WKG~s@B{!~O&q1eEVG|_`)1q{IKw9T;V2hD8Mo~zR+C_A>_ET*|b5WAsB;M{^Rg zc)mj(O@pqIvn_CPkX1`U@%w4K^k1V@6w>HJPbrY3-G%6Ch6_=&V5WQjewbF*LKT`# z5FpP`sn{*pHaD+zh<(+t7w&w=xsE~V{$H*75@{d&9Ri%Y^itvG_?8QvpP)I}TGM82 z6DLO-2Pb*k37q^JelQwj-QYC0AZ(XtcQM$(WEp}MLO^~y+4Rh1Rb0Ul7oA57a6U`~ zJDsVe1V+@rDb*xwxsV5dZj;*Ncs0Zm8>wYm#o)x`A7-fbnaqszGhtl;6E2;@9*Nr_ zqsuybo)^hq9Q2QT0H^_@;vGAiNgbiFPh42s7pyp=UJ%_i%`!&9Nn%p$;l7||R8~;5KqMnQAVc7c z!8=2xJ%a!Y?T{tOy;{T4f56osc?Q4{JpLwLAYVAJFW`#$$eV`tf(9gW1cj)@gcMSq zzt9V@@pydXOrD|>k5D}Bx)qOOtfx#=JtQAEscpQ>mhVa*IGFF~@uJxAg4p=IPP=Q# z$ckFAz=_Y4NLnWQ^##xZFsv&iea3tcq$}>6RTyh({+_I8EA(?}k|$7ohOPm+!6n8} zAeESV)b1z{`?a2Z)y5EnfSwH)E+{H&qqRzW#mYENWdE*<_&DK*6`(CO#5A?A+9yH1 zVPGxru-gIM;Q-Cxw#ZOtR){|8roWQ+OnsFEpluh(f|cSR>K;yhiiq%mYJ3*+9|09^ zMr8$Q)=))Oa5&<;f@;x(jM5Stgq+LWg#dCs>xD*gKAM)=cn5`8&}UYV6H8WQFWw;) zZ({|tE*0E1Du5NxMmcB`72pi1N`=Bu1#Fr?vQz?`_$#%AMihjZ4+U$H1ehrr%qZd* za~?<*zDqeO2OF|yDMURYVZ+ySuh^Z662sxVM=ErFkNGgbR6KCPd&Qp_5hBq8PO*3v zL(vlLMvpdbp=s1l$IIKmP^!>B0S`<$$`BPi;i#WbYz_{T^jU;ZEbbOU;v$qUg6Ms{ zA!KbKOvzC46-XE+GjKGJXxQNEus?PLgm8Hgsa|RzPoiGyYQ7G`#4w82y#Q*06w0Ux zR1Qt-+2~HpBtKHT4`WyaBRW$Y#H^#J1hu!0~{&>+ZajM0vC^R zCW#k}a0wa?O{fKz`%LIysi*k-6H+A_6c`5{XbJEd-YEVvu_q88dYh9c@Cx(=O~YE) z?>kWDOA_Gw2ahrq^r6wuKlgr*pR*je!NlLk)5r_N)+O)*jK~WR+5mrs3cQM%*9Lxq z3b~BB?^e&AXOWfzG=5E+ZxNQW7xc$}7Rxscink;WX79l`HM7&eyAL#a z>VZKfGllg)^V?pmv>>X%+W41Afq5r*mlhWAR&}obzI2)+XBh;hX$M&p z@WtQ6{Ho-oQ~Ay}Lgj&JH;yu5pN4~29qK(OT~6dZFuX<(|4bz&p_OkU#j)xVJz2;I zIrC+4D8Tr`5mR(s5+A`_V(XHJXi``5XF944U76`N^wVLk^8o5N{{X<&VSJ7DJ6^IU z;!e?PG=Nvm^&YbBzB9*0P5dKcNv*71a+LWo#YHhFZsB=a)u9w3QCQDm*|{5`rPM&y zEAoZ*44BILBWUK{xsH``_l8T(Okoh0n!K<8`$rhoym;+fH^P;{XvZr3;t;sAFl2`} z{you~MJ8{3jnDxPgg4;Q$Yq+M;NWAH6smNM}eV zXJ!2Xe{AUGp7nK1oVs7heBFu>;1oIAj}2J)fqZwLo0Z`w_yU>U?-Gz!;gR@^*YP+? z-`uGrMC^FbBhC&wM;!>vuc!OWIW{>f+Ee!Tq>V}dBXLSl_AjB3@IgCTGLLLwO-#<5 zHVPm38;W{6;M)jaKGYo7T9BeA_?c|FDu>%)=ThZdW(H~xz3E7iuUwtBWDW%TUz)vL|Go<` z05ut^{NTJpox0t9|B)^Jr&fK5K8B<107$3G+6M<#`OTHqqmku9eoUpBlvwgS@x9Y& zk^n#-2R(?J9>`Vo?%W*L{Ua}L(SM;ljNa0zTHEY7kN&hwljrpxL-qF@o-u+@s(Tzo zva>SRYq=A<2VF4j^iSZ_2QQX(ogr!A@x<|{`^dz{L_M^ZI=WE|YUm4qL1ih-lhUel z`gvB`%F}oddXjNb7?Z4K@y3xYHL5+dkC)KPu)QJCz2&Old{Vv8a&b!Xxdps!)~Js( zUrX8OiKvCg&90l4JgDvn=CN4QbAbhh#1E`Y8wTTH=ztk&{i#B0F(nDBtIG-vy|Xck zW-%IM(9Ob(l>1CH%2{`_4-?GFX6+&m4Cf(i>A^A!EMh^!A~YINXzQJqX#-3&LZ zTPWO;0aq4gi^qgFby{MkG)t8%h^E`6H%*_zx0dtL0sr(kwwd@qi=$r%)zn39z8U=9 z&~>*3L>6J**eSmpIT#`Ilx?DPJ)?StD7A!T<`MU)WnI*V#919D+J|wHBR?bDWPy*l z;>m*w^D1FZ@q|c1yOp zGAlxB+ydqq?W$JkaYPJ;=+n5_HDRj5-5HxjuP{WPxgz8O=s5~cU@**GbbK`8pB!*_ znUTt)Z@Ge`d?Fj>3xUw1f4VR(GPLn3weVRpD$oI|3lh@Bb`Gy9&my)?bf$G>G_}*5 z@fc)FMJSutCx;Q*{txcZHnNw&gJReM-}u0!$?N>xvvWJupV-*-P>$&7F$ z30WbVkQrL~6HxG#va+PJrcg!igybadKxT=MmtaX-X!8Ka4``(A{W1CiI9as*FB3pc zskA>Em>ifP;d)8TknPt$3#KE*J%`p9Oe2T2WuQ*;zH>L^PolQmep^{;(y5h%2j zub2zjQ#Y;=dks6%Im?#Bpva=gq~@;Ey05+JBTfW^JG9|Y2d+Yryp)Or`2fXZ=_A?J zhBf)V7DAz<3c9~THzxWNOrnfjSz#m9x4-72E>Tp-k)ea|ehAaF*kY*Jueq))OsUnE zVU6Im<(5#wWj_I2s7o$*_8(%h6-GR(R`T@3ih^bst=I>a@>cu%J;MRuCgly~Q2Mr^ z|CAdskMKL2PlLopW{_Dk*`ReNVGw$_3@awSQPQnbbR{%Jp^qZ-)?05P_hn@!Sb7Cj zTh{HlOynlfJ8K1nTG|!-GmfR1U@_3ct{55f zy8C(IyQQ^%Gs#u_k@;%|*TZ;6a>EIPwvipR0;bDuzk)4@=6HD_IZAC%hXV8`Aps*B z7Yw&awtYqbEK6=sc_-4wBpn@7zurV6veo)JceeLy@QVuXU%y?a4gJi(K=99DD&OLw zt=THGvu{)1SJMT@q$>W$E=ms`$mmV8b&H1KbY34e)P5?PW88RC$@qqu!vRAdP^Ut3 ziKZjQgsbr|XkAcp*BI8jOm>&wK#EMt*{^7Hq155+X1OcKx51X*xB2Q0Vz|Sa)$Y0t zWGw$wws@F{s2@ttwlMpm^E`7y2| zA2z=Lxlj3?9uuizFS1>tNUT06D^O;RG(pZD406Rr1N;pI0f7WJey@}34BHG|jY!d; zW(rTKmCVtAb|SdRUu%`Ezdd^(Jm%3#4VKNRWuSIQR4xb|uDenQ{iw}mA3mgbN|fKT zApNeuh#r)}wZB^PHg=kd@K&LI7sb{BcKHrzvvwXS0SZ3+xxB=1}+ zi&tPeqxE^kCsN7t3+T?7*&m8tW&WU|-%v&_o28=!6txOhSzoq?D65bnVc!;5n-X3&S14?JE${U+*Vr=%tbz81Awqkx2V=7KyxzDdGvsw`e@AZ?=BVW5c@XX%n zo6zu{Gx?KBJf}Pw+Xd-NtOIto;9kiH4^s$Ww7lU29Z4ER+(XRPZR37v-x5;eKYlMi z-&}TE(rBZvr(ful$LdlO6*?{E30KeCay=1*;p{CEs|!XhlSbDS5fs*(7HG?D^Isq z0&}06GQ3oj-MYqlW4B%T*K7{d4qRP9C3){vw{;M8%WAZbf@^SmI)bSZ*wwPG=~XBN2)8Nnq@pbV3WZ;u|_z#=;87$ z#huIbsCx65a^w%rWg)@>dJECJk;5erd_C|b^*6jAo|+%?u5<2+lhZ@hPdt@V=b&ub zo|aP+wP!q~*~O*8D4>piWV6URm^$5$Oc!n=8dp*D09in$zcH9b7;)wAvP+tX(enTLZYf7k^&?k5CIr^hlf;?>vE)+N;E@CZY3+h=2y7#Lr@WQsH%gqr95Fi&MeU-dY@e{AG4e0MT5Whl zf-!PyW6@P0GdiMm+!dcj=>43|d4~#H3NWjtAMP~5KOyuCqaB))h5@BisEP}U3jTLL zPlOh|o80a`CO93_UEg4yJD&lb!$cd>2~_Do>k%Wp09Q1_if;(4`G0+OkkLX`^7LV4 z%wylstzF>opx1H^%hCC6#n2+_yl-aIPKNuY=6&Hgp3blg!^yJtIMhzR?XT(^Bor@% zu;%tW*K#!2hS$hZW@*Uml*v1LSjx}$A1T`s^JRyvZw{5=G85|Hf!+O=;Hz{=6>Ww^ zC|5D(CJOD$@~2Dgiu@K+-|CdwPR&Q2fxp+q-V`#Ii0^X@E+)f@y*|n{uFuMqH?EMz ztHy0F5-*vvDRL@+ZkV%zDs=bBQ}uM+xv7`t^HVtvp4pD{%wH`XBpg(zZ6(7iZ*V3G z3#GZ$I|+5k%z6z_TRtrRO34u-EPt4Xl*rUK@4~SOZ0~GuwofG(@q{X6GMbOr@d9M^ zf6HNyXk_4PMjBI?XI6noPnpAWFp}sVZa-IF=g!puc?x0FZYz$=ADarNN4MQb&(b*w zy@P(a>A${WKk`5FW9aRFe*B*O#&*YL?HjbTEjV&>*q|+i!IcH&+TdeAGlvyG#-;xD zZRe$aUHz{8V`9mcG$P{V3n0r%@T$YJJEPMsW*{1uQg}J4TwnaDNl<3(-*PO+F4j9A zUtIJsIrj{oy&!uTXqK_B!ZDK@#^7I9K%}K=L)!G}z$b-Lx6``kS>T?K&_lgt zYK=F1tM`Q&ndE8@>KJXRQHQA-^;o-zSycdT&BgjB(;H;!+59|A;K3hVn{$1=xYmY? zN0riX%7@5(ou{0V_f*5s9CY}}W<3gMDn~{MFeC!L|AMB(m4QKJf$FgQ6HK+}OhG^zWu*2ndllU@w{&H76K>%UGJ4sT{^%@~5D zCj}=7hF&DzE7%d)*#Iy%XCI$-mmdREQ#Fl1A}0DVOr=21sXpTL=0Kiton%y;F!3>A zf)U_3Z4dDF+Am6&&sdGg5s8H=&j(I*1zHO5Jh~CHsyu$x7P@;+PY%M}wi`{v%i{dv zCXHILUJuO}T}PieCpH_KQE4Gm`N1m*-~J;#2vzVEQ|pQB3AD0E7Ae{$l%ckK;AO$ z7SwJzAgi;a|Im2;k>gA2iFm;3&H5yFgzk?Ems}aQ9|2M~Y0~dW>6Kw4#@Tis-Ji5~ z61_q;5z1COe`xN?J`*`te+Tz5?x;pDqL!?*X5u7k_R@OCnTFyquDH_U1*C{0Pbr^a zC~L1ZwQgAfDO9~>UXe^RVUv)TLytcBJe%Sv>F)@BPocg!P(5&?7CvsEydj0a&Dbu* zzc8B^NM&pgQ#!AfDfK)l?U9C`TJDkr9YZ*S^v}%bsUS+UF4HO4de1w_La?Db>$p(+ zSYFUyt-=~W{PPNsjNYVSH`x~d4+=o_za2sz58V-SGu^SPA`u!k(5c{He|FywVHT&RszbOi%$v!EbTU`2c zWJK;w{k%02-7?_6a9U`Hox>VbxI-xvF|%E6jSR+>3sB%G6D3b*=1Zw{X|n+aYCG?1 z;lwU+PQI@p5JraTGhAA>u=Yl!U zmpS(q;}2DvYZKNvVe=KZ)LW=^)a*I2mZ}tV!;bnDM%Y!2d3Y18)zP%c_I5|4XIh~2 zJ{Zy%j@+Bg8EJ9y=nTL5qp8N`eKjDAHfD2Kc3E!rz0W)LsW@Z$uG-e*To7wU%{}e+ zdT6?jt)o)OlsG`r*@}|%>i=LGS`0| z9RPh`#(`<`U!ejX@Uu|a^%cK^qT?|JJ~W1|S(cN6rB$Fk2TT&U{e^E0fP4d2^zo>1 zkVn}1qyLgOm^NPmeZw+uOTzA0u_d`wm^%t2o+`yxViWT1FKt{JXppcvt85(^ZR-fE zII=;WAJQJ+8S%q*t0Bdoe0Bcx{OOK@-wV`N{IB5uYOB2c@8zgJ)%mmYrzd*N&rbE~Jsr@>{rm-i^JNKyJXLz)SM0jV#!&`jOtj>9w8rnZ5@|a42qRX3c?} ze=CFRXtqn*JD%D%7QUbf7poUiEct36{{8sXe_XwKfBE*+yPJzQ&&iGdswP8AzN!jS z;TY5Wy%_T@zUI|wU>|cI zRO)byYm#v$%A4LwI*pBa0S`L|5Br$Q!X&B{G+}B^uB+Ts&<&>HX-W6Px-9b#L>A=3 zs>qWM-2y>>r2Gt8clG>9L+sWD!nmEtjSbC;*^l~Z4B*ma)$M=>3qIQ$fOfNHNZC|# z_cY3USpCTJJ@!Vww$BDIk>-+%o?Opnrl)xn1BHx79uWaDYpz}LL|(`{p7yFVTo}t9 z`ByCi|L*y}>RsqeR0SjN8zxre`Z;;J&jQzsrmQcGWK6{3prG2$^E9DX5Q}J(IJikv z*+~tz4FHC|dVlpo-pbJuD+)nG9-|Pzt`TsH{QRM|GPT< z{b~2s4dwb}s!vGS*{GX0*Tje=rG!fe)>LK64;iN1z5GG?me9zaqAxU@C8*MbR^?h2 z`fdPegX!F{f-j+XEtKjHMX4t>s6e=OT-FW>YHw*iBEji}ea8k(LwQVEQoFx#EqzQy zjh`;}$ix@dA-oTo{vyVg(K&wSmA^5acFWOySmhln?l`+Yzn6uUcxRd&&XBvQb7*UE z{>zxIAbQ+<^YWYL?(oCf^gW9qne6Up=LA-OZ4RI|-hM9Xk9xDCi&@M%RXk8qNB?ap(euASV zE9e&nNXX|yB+RYEWqMKnS}>WTyb^&dAkko%AOq=^^;VRq$3%AivB{3P3rKSY?=p^` zJp><3G<)5!y<@tADe~et$OZ0Krj8k9K1X?s8NycE#E!!Tp z(O4_uv)Jcbw!I$UW&@n8YhA-a@!;I!_}n0mdnEq02XBj{w!=K_bLMAbd07Ofirlw2 z4mF-T0q+^ch8WHi$CZXTQkWY>aiV>=5ITRtCHNcr`>hExLH`kKuMLrXKpGZYZ~tb_ zT%^R#p8^m~$@GEvqa%alP`x*1$FmJY$WW_{X zZ`OlCniOPyQbP3E2|8%~vWq!a|O1v{0zmDOd$i~Ol?vmn*L)QV2#jloV)Eku7!iMx?Q_PlN1B# zG5xrEi?6XGxh9AP?^Va<#+B6fX73d~p=6_f(H8)!SDVgBn$Pq^_tSb&`&7J#&uQ5! zD!LqmG@Oh*Xd`UV(#sw){q_bLUqg={D1d5&3AkPcBsqTG+mUsU*5RTGnbQ^RkF06e zS8Kggmt@7j`HeyI0%Bv}sZd=fTfRz@a*eazR+$ho6F*D0gch}tXXO{WuV;)za8QQXIK9I+(xp$r3UIfinyw zGhG7Dei8Dx{2|!}iBhaoV}W+e(o{~DRhd_1L-qt+$TZ8fpQ~Hc8M&te;;tlQzCJ9| z_!3OlT#Za~kVw-kn5@q(bq#n0cJi>b(Yv)X)0IgnOeCjb>oHGr*>l+$cRE;X|6aYy zs498#hfF57dN(4|>bl-+tW=PT@2|;Au5Mk2|9)jnFTd40i4v`9W(-nTKxnX?n_X=n z_A!?#oG@$mW~^mHAzqvtO!_6gR}7cC(96|-W_|XD7!%Y|^`~BRZ8tt|^O8XagU%dq zudN*?qNyV6^j22no?-w$P?|Zp;rVm&N~j8}`;@KE4B96kKh0$^ENZRYHrx~|+e?JJ z!i2X0N7NDfSblwUMvgu%mHv+?m+ELvE;o6`6JBabMJ!oSAhs-+c_9dsFMaXkQII0| zeKd1v;W?G$UdzqwPIE-)4e5#=dZ(+IcMu}vn#c|Jx`M$a3E9S?EMt(O9nuxZ4vxB4 zESh>hG(?hM^1L1$2zt3B5=%^;nQE9=d4MbTP!Qx|Yrn23u!n?DVA1^PJY{$1Dy2`) zUy z*wbJ;&6pB8B&v)KiO?~~4szHxY{QBv zkS7u#`>VGy9U)YBFIkfPCq2U#_&eEXXyly~EF3)Wp3xNAg`VL*zb|;ng48IA1!)u? z>fGB{kbS`}#`keS;EaC2@S-rY$mD_;WPgZNeEa@#)FQrpe`(jC|0$O{(}yXB3FX5~ zf25#V%c-whGHT7rO(I-y_H;+ajg&5fP+xtVe+s$s&^^E9A{bcGyv6!yNH0(7^lMOh zuh&!)QnIPClIMX6B*db;Mlh(KYOzNvoL(&x5c#vld)x^@D4i(S|Zb#+bI zG?EWPJV7(1aIYCB5vFD;R-VqSbcjtHU>gpmMx#X@C-LmqK*@>z+d0@)Pv<~02j!kM zD|Q4X`(m$9bfEk)%!@6>4L!#r*ETh{El+LMI#m%gCDstvdA;)LbvWn3L}{pjoKrp5FH%h%-fwUe+tFJj1!0Mk}z zUp-bN<4YFkCLGoKQ3@XFM7pz@WBZ0a-S_q^$XoAMN)nl6ST!Mty>f{g;Xrn%Zl;oy zn&18lC~rnhJI4b!fB)*@<=a>DP5R5Vyq}eFRw*{)Wr#CIG)jw%vx;dimIvH>q{G}) zuJsmvV4)S%!OE7L1+wmZtMhFGbXblg3g9WQsv` z4In-=8z7$sDaUY|aKLSQuj_Z3yM5jGeMo!EP2}z^+Y;zgxRiy)&7z4*tW01N&MU== zIez+c+1w#U2a|ozdg`$BZ|!@w?AlrLv-1as2@XnN%^W`~4s>ocwwwIv_ck^MCni7T zbB^YHlz85eoMtekV!wn6eOKd=UO%l7Q2H#Yu(Do3QwdQ@1)O-8OL^!_(+wA&4K)~+ zF|Ek5xBSRO+P~{9lO{ad6d=uS#tGwm%%M`gR(g=adZC*>=+3*<-u}&2{Vki}*JpWJ z@BioHIBpu{!UiJtXhu2fi5q2_8=5Qvi*flLd~W0uNPnND-XFy`57^E)8E=0%q^8Rd z(XVW0@vymSO7><(llc4@`-<65#wOs6bh>RunFd0nblPB1S z@wdPIEs+JGBxMQTv>yxtMhL%t`gBhI-^JTE5dN5#3L81&SM>u8z=3vqdi?wKH}C3} zR?Rm-YjoS$2pkSi^EY3AJtr@D0fp7M={jR0GHuAPkR_qe-Vem2ux`7?I`1X4|HCu^ zzFL>2@rTCVj3*_QgGJIQ2b$v$8})~CaAF`V_>5%ymXW3UVhhE-OS5xQ-|63X$0jdR#xViAPiH_nJI-mZG$j3 zw7AteqM^uh$U0H3=tRLXTJk&eE^X-6&*mDO>+yVkj3h)BM3uBKcV7P=haSacdJU(A@ z$AghViQ&XAz>d&R_L>MaeYbBg5R-1k&{1!QrAsK zp)d^b zu=RkdA1keb6kY>iY|C|UnC^a|(Y4E^KN>l>CR29B_SQk!23*{A_n@3AV@(8C`A~Nm z9Nw{^iUrbl!WGa7IeNx?X{38yQ&r$SL^H1M*pstG5E@<>_bg)iSsse z*!~q0ii2Fzs<@+%VQ7Ea7e+!(U&c(G9{s0x_58mZPVv;TqJHCBk^AqEgl3s)*Tb-6 zKP;nbkf6dYET(adgmyZN`n7IHU zw40nQU%azr|0rs!86pj^rB8-RU)94hBFnJ$562smM|e?|1z%KU7-h%wGV|v!c$ST= zY|jYf8m+nm8Jn36PP}UtVWD;xcb72MLIKDF_+y}M3rtuh6?9w#dfaT%ho*($lL?}3 zM1lm&b6>GGmG|PF7U{)RZ~j1BpGQgZGDQZGR3(xos%EP0|KxH6qt@RAJ-`)*vuQ9gPsR6mjh-! z93bpaAtZ7}b%q_n)fsB>8ec)O%xS`c)wt^67;A)iIew!ERCluCY0T5@3QToX(b9Cm z-%$AHYrSth6}GwtS40bQCBtGStk@_bmuHQ*yra9P1G>8%BivHNB21|GU32V~!gHo_ zjVyKH>WEbMdejbBxYoF{kFeZ012qBhY8ics(S`QO0S!LLo^Hee`NNknqON!CK0I@l z*rxliAzH-T!eQZ~EnvhzjS@kcVnzhm826#)9E@=uMYs<=$s)=*8{Ko#U$pzsW13gSlep_jjMxOZ@ivi7|JzXt)oLkLUAa4E(4S zhN@i7{)I4+$TZeVt>;*%62z+76y-ZlH+H_G+>$uI2W=5!>lD4-Qz)uMAy@IpEZ7xE zjFyQk1Vu8w;ZQsvb&b}^3H~@w@+yXAn5Q>vBa7`>-LE~#s|H;2#Ayt@EEq|uqF|!T zwvQ-txt$wIX0J!{P-=G>m`@!~2U={>l4ACdc&^JuyH|wVe1C#Dqsb zJ`!C7fd9zeXc;Hyj2hyS@w_pbBeoL$>@;npN-06lH05NLB$ElvBFg2FrW6Cn^UB^; z>j=~+q`!#LADBfm6*hR&JkNLn1@S>sg=2PTa|1PfKk`krA(hfdE`@m}2-VyeH^F;b z%O8rPZ~*|D`umVfaP4-oHU&XOsZ}pk1ZG76r>-$q-& z+8shAOd#4Pxjvtzth1+tGtAs<#9I`1J93XV*_&4x+6#*7UCv@lsEy-t)ZaRaZ=A8l zu)SoPn940Ys#-iw12F~?L1y~@S!0j&@GaO916KNPjfMsKwMmCbS;k7pU}3IoKSs}N zT&*Sax=UzI6J7?I$`v%Z)vtoj4yJcTl2a4j>>pb*#iv{svRd`a zZ`8e=WK`MR?SyNzWJS`gpu?%V7+0sbG~OSW2chK3m0m`F(~FGq4LMPakyjR_@h{h5 zaT%?VMKe}Ry7o~5Yk_%0dC2E(3h?V4=$z!KL#xg|_oG7raagvu028V7;yNy_E|ss% z=IvTa%h>_p#YW%rjn>?dsd@RS&_ny_<^Sov2*tXwWL3DXM_N!Dbw>#z0NX5Bn%2E%(3Em7SYdC))psw>2%odL{a-<|Q1 z*f1{=CN5{wX#z6mkXGVt7@RZg5!6yD=j5G?^t56?Hi*j5WesbTxvwFyWGP82kj7)$ z(&LD>%xdXw)Q;ZNiT_C*Wea=JC2I~lFEv0WN?tHq*FYn<+2~3jVmG&Z*f6bW_KKCN z4$ee=rU|2wUX0}im>nF$AXI9M72S#f9-@+lFKO;OwD>VDI_wAcR;F?OyjJVB`sF(L z0+V|bn78nOD7V7~pp@+%w`5K4_O~?G((BB1nt>_J;B&L5F)opah(bF?dfH<0RqfC; z>N?sj)T;dB-ZiA&#BI39posQVb}+6<&~x&uhJa&)VW-h!Le7a*@#p41Z0)G#<1r3v z()u7sX5CtPjz5t2}}(GR@FE zUdb+P8Aoz zQ0~miuY}XNl8A_ou`wMGNel5{aZk6X(Z$szylnNt75op&?W3KKzAytx+b1=C-O1_>*jLz_S5Kj^QW5#Dd1^#>(F zikV5)cg`ClciRYJv?)*#C)pXH4QA_uzqq z4(pX=vf>dJspbd37Qv9Asw82NSIL+S{Z7a=3ozTWp=GjWiYOU{?u2lmN}6SLuWo|? z5+qLJOY#NLI1=t<-x~{Po>60>T2>;#P)A;F#{}u^C<PmO?c0n3?bmhLC_KyLCoIxMK=}fo#%Gs6B%}R!p#h zmf-j_6+HQp(4DC^uEO3M;iCaMneCB|q}1NA8wg-d)EHndZ{%@7VW?B`t!V-8_>0TS z?y(QV2|;K>K>GlCZj6awPEIbeeC>TXlCe12@p`!eA2|bmN#kQUE}AViF-6`@1;8FtZ7{I7QOccChufv0u?~ArevHAl6L@dG`^SO~6e0JAQOmrGXQ_g%n6<~OONlJCj@N+qkCllK(0_-F<55J+ z?d+UfGut!AUvx7FEE=|^*eNS1&s5DzZ@T$+G|&;eaLl%P{P1UQ-ExU=<1JHleyJDy z><8q#2cjKTU2fOQC3(*hdB+MwfL9-LDioLEHRV~XN64Z|?G%Dmb3U*)i$4F-dajE#z{rLIb zkN39BL+gehXO2HymshxZLmEYL?f1sWGx;x%+>Cl0O8~^x zq-fx?ebH3rqYIXizg4W*5-k_ntdQ6kosHg_;@!)5J!(W4BiXSOnse`3L^MQ#+vi{arEp=R z&RS1y**4xdNKD|hX%%e2V#YzEz1awm!zF#2V=Qvwh-zQukG@t`cs9_BCazJCKd{8Q zITFYq*V`uJ7f<%Fd*QHWcEC2cV{aq@-?HsMMd#gtRjs5@Yo6Qoy>QPBHjwXV#>Zfh zzd}@YFU1*oCrkaGSCBz65-;_5DVcgFOL+g291<@7fFm;gJq~^~%<_Ut8zbjY&&9=3 zy2Iy+TnZu!v%7m-0W(RQo^>zO-tdtSJLYE&To#QJe6XoHANWA3&+b4tU_XTj5|LsD_doaYJaJMxx*-|Z_ULtekwKQk8+l1Ngzf~t2 zMZuJXNtoUVPlo}dLYa6y#lDsN%sZ8Ie_MM5N zoZMA3w&|gmwd^hkq>!4lBoSTvbhjQhac{2ZHHpmgxl+v?s&<#Vlw`KdRz-x34`~x%GcpLs#kAo z3b^T++fXM4#!Ygm_sLWYu$qDAzg)7Sz_VLyyD3-HXHv=Vq~S|vcy`?}oMW=%OoT2a zR}8PMDrw2iupiNag_VxXdsgEkvQp4hb22Y`g%cids_glQED@yfQ%iyS6=xJ=&4+i*KutmJ&1gR zL=%$+WJ$P4vMR;+Ata%iR>jcyJ+^c&X{3JD%78ZC#f28d68J8{3yBpDXOp-(3!_#a zDS5{z*)uquzylHA8hm8gTg0N-kTq$VA7ArD!N`j>6@q16A!c-q_0E}=B~8}I3n{^alT?y>27yRg zPKC?@71J@tTRkM@C32G&a>H$uyv2!w3-D%Duq9ISgtJQM(EAB;) zu4o?n$P!a*CC{oA?=6k>N&Df~t-UJ+MZUvF>2<)FLDs=r5-P%^D@t{E zC}AM|f`tKWaX*lu-10EkSDL%f_&H9)vLN`| z&L-ANHPspBUW?Z=NbNZz6;FRZWEC4F@(da;_Hh?JcKi1w6zq*_8)TksSv1U#M#%Ck z75&zX9t(!Kvxb}6Qg-$O#<(7!Zp9t9iJYXUgy=uCDAA0TRIzf%9M_c3>UT8t@b8HE z6ZoH4O6UN|FtumN>~}`Y6AdU!S-6d{?S{hi*M`@^xAh(uEE%YN9EJU6M&EvW`Eu+w z>36Ik2>)(wRfNY6+&uhmm9K(9iGQ~_2G(J#9K1!Vgw-a~mt5V(wc*-%t$DTn_~I2Y zzeZ{<-FCtuLt#HySyzldu&2!`VYMqIw-lUG7j1*@9^wLUY1!;xAGd4;v+)pM(#1ObmUVYbbbLD&zC4Z)8^*-P6t(IGA4 zaqOizElN4fkgbI=|TUg9c zm@R4r13KHMB75Y)&EI)CN_08K=RKC{o0qO$*3PN{wd9X4Ud_q3MwKya8L66Hzs6~J z;!|#H)YUz}LJ*_Rhj~Ct*WdLy{1h7&wkHg0CWujc%T&C4&+^`I!e;%giB>n+*oa?ZF!GGf^bQ}c)|Bfp8cn&j z8&a>#N%Q48I-B4pU-$zHh3(xK@^35{UM%PjSgyS;TsbQ?9CLpf(qdRg>XjrPZ-WmN zBHb117c=|2@DGwfO_8nw$lW|!`*f2C<^`L%?lViCF{s{cd2PS%R@9~*YniilJbXLb zwnF(lS*#ZqU))BhG&M<}Xl;j1!;~86p41sGdR$dh?zc!btSJyP; zzt#V%u5-}~81BcrZ*HJ*FZhcOR&?}40w_VqY&HXv(#c=7%$#PY_#pXu4+?Fq?M^E6FmfK#D#_A$ zO4N3fadBHWFqSi28{ORI~&`R%KY1%X%C>hjiyv7{o z(JM6X9k4zN;W1YCu;KF!Kh}WdotAs}vBEIT%trgM!v3pI5Q={wr-2*&kK()Hf|s0@ zvKZwzXuRc2D3vPZ25%UcG5KKKig#elyIk%9tn)%a2ZBAn6k018D%{+Re?B9E-J3aa z`#$+sV^)N2AKCAoN4GDpTNT=luf|rCNFi7<)HFBlNd6X2_W0jK%TkI<{l68>aQogp zgqg3o{qeJFyk;j_AHzFmFR9c(v==P@}clRW6(d+;2XtP;Ms zg7dt=<9Oq8U_w)RU?(78b-YE(JnzoOpZ^v7jlAIsLUV@t(Zj=BnYOSXxm{_cljnb- zX<9r-9=tS?R56$^Zd)->;k?)<`Hi;zkTH)TYo<)-{ILYFcV8IUwHvK}c(^5ih4rv9VuxY^+UhL&6k|c>YU} z?!8OoFw+D79llzl=$l8wf>0lZzCDPd&;Qmp)tP2?y?^sJd0#_8=;AdmoFA%U(we|8 zj7dT--(COZ&BY&Hy@7qh=$3#*vRoUfe4n_-_*>g|zXp#@#5(On%UvhPyVutZPu-43 zuHS;v_S7A!I*DcP4m^^*)2lz-P*U}#Ky$ca{+PD8z=Fw7VuPv`6RY>E+A4FQzQYds&Pr($RdS6+vVE)=nbr*G4|C zyLSzDuhnm_?+%6~VyCo@ynmP{nBWR9& z!5A|1!POLpg3-#mOOH)O>}>X?B!&DdlnRX_BKB!C-eZ%hEYqhVh2I9t6jtaDx5pMu z8=7Mqc#_I{aZigh&@4JC%>fZ{s4QkP(V>0^g6qJitN+J>m!|f!sSxey+F@6TsGX9U zjA)W!zT(-%1d&NeGuzsdmG_K=rO78xo}lgV$!~xATLKNjQkL*d8wkOte*N_6oczCw zx1o@JV%p^@tXPFJf&=jZt26TN*WbLW>!g~$1_7H&^cqLpcFxI5FZ|B8_0e)Y*kK9E zB`=p=Xecw68oLw=79o0AXJkZA+op%eTxQ$M$aL4Tf#gUF1};N{ldtj3D+d*i z!4#0G2@lPrSiz|c_(!B4Y{XKR#k^a3peLdWwjYz)NJ!XllDpYq2QL zSvn`=g}#hSp3pL?nW26=Wof15YLg?nW<(Y#!}9n`l6U|O4~o2AzzMjX;ewY1Ew(7U z*t65b3i}g8?l8;>yEQ`gRao37qZ`@*o?VaVFWI6`_$r8ayO&FiX=(=zbC6-Tk)NP{ zget;-=SD*0_UgGfHCE2G23J-z%eFK2QIltW$6}a;9hlN;`XR9L43Ro7Sc!Fej&L19 zLPo|BJQ@ia8W&V0Rz2tg%1Scm{{Y(LI@J96&Plk<_faz0RB&I6j`?bn<0(YkX!Qh(0J z=+JpckIn-QY@a@zkJqX5v3hkrMz_vK?AQ4y9XlVTXXgQ3JNN9{c>u2O*Sm9{?ww=% zckb1}bFUtr`|QCUeLN5CN9*VL$Q?Z&qo-$lSI;pZP5;ja=K}w>bw>S_p+K1G#!~L)kdrAQB6HHHWISNiiT<*cWfiCyR_xT$M2!$+C|0A zNGio~Y0NvhOtJV=mB*|2UI@Qtrn`r6qE0pXisM-7Oz$Yq=pwUvK7=}@xa{LA4>@Ydwa- ziR;n8v-hBR+!^-~v|{qAb`?@ax80UtAVufj0#^Dccd9M#%Znr~q#+9D#E!(QyRmAcW zyVg2&Twf0p>QM>r8)Nnp`f5Q6zhb7iGPQO+FG+pPlURzyu`#cKXfeSZ_E~IEhk7Vh zRyj@=N(|`LGNrH~GuhDMmZeUzian((y?M?&QCOx2QFwIQD6koC$UUCVkCAV76y=P9 zG0L{kxWaE~uFxW)Wg)P{N+=FnuHzkYIA}y4Er0A|e;#Rj?z20ab(-5W#*(v_qP~gj z4cs~HV~9~{G#zqLE29iTJ@HeR0B1S^j5BJ51E^e()F_xy0ac?Qy5OsU0INML$vweX zUcl8Tpk8p+Hy(0l_H1~+-oV+y*ZzS?iEdwi=Ri!I| zx;N*<0SmJRM66_OLDt-K^s)_0jAz2;)$t5sti^qdwLMZD!m+0%G1j`irx|dth(zS- zK5?lD9AgVkM1qYeOM&ClnHue@w(W^4GuM#hBNr$OqG>cJKUUnv@^x{{f1KHC6yhIZ z4Y$Gd7gv`ftqIte5rO81W}squ*b7+L_vG%_{3Sj%c>w zA808}%0kV2>*k>GP9c*&420$p6*7Q7Q0%BZn$-Qh<$lJhiq@F;S>uqlyiEEWoNlu~9@W#|MA-8EZ#(Pa~r(>#=u$#L_P1psl^~3P_{P5>)JLtnx#=Y5>lHriuL> zp4}(fmi?kX+*Xbjlt+eh9PW$8ahOG>!xCeyE2B;+WQjSmV$WFRDrAYV)|GKEA0V}D z2WC{N+bEDjmKZpzQ4ohMF}`(W6zG5@COThk7iLuc+-Qqb*b+mMD7SJweqNRsPrR{Q zvtgAPZ|LwVyjjysU3)j8u0?woXO1(<$fD6;>-cy+KOS>Bu6Kb^+p2Oo`xnAQB2$|K zwB*IeNopazwgLDSEg+N5)-(Rb;QGcaa4*(sGqPVFEHKw%va9+ZQDv(=TuTxoFV`94 z8Q*Y|4W`K{QzQKhCkSAk(M44?rv1IP{ zFFu5%9?b)Foej}K=47zZ^5fEVqE%@{r%)W*{lDG1af-%`~Nj^q7A)(i*hE586N3#h=3dg&TjW%_qt~N zb$ga^3IYKoL6iGWbaD4n{|3AA_s#Do(f-D}TRs;T$_U_C_GP4v8DHu#FPlH6w2Vo4 z{Ol~tKSit>mT-@RIt8kI!GUUc(hbu9#Gj3sYebY9#-Bw3jg3+R7k4!3Pf=>aV~SFn zqSU4+wJ|WKD77g{ZHiKxqSRu>6s0yrsXaiH8Y*oav1*ZvGyS>hK(&9QI5njaEMXT( zA}dke$XgZ>fYP!h>_C7PQ*GNqj4c7I54+$VH-wE=Dc5TJ9yn%S^_x42l%?(uktk-U zNvT&mCl{n(3D3DHlR)uz#5z;>IF{b>Qn74_QV|t(+2q1gij8u3z?4xO0*Z3a6(edb zt1Kl2`>XLs#}a&8|BW}BM$`~&v0K8piPD^#-ZPIur`X4w7l7>7ERiDJ$8fa+qZXOK z9Tsh3Nvp!8s^8I!r@Y*{h%mffTK5>N%w7+Rn5sT=Li4szSIUc32tr*rd z9#uqanY%9}%|7zMVNre%(BCS#*p_jvI?S`p(tQ8d{J7ITuy@{GTS3!4uJ*(YgeC)A$n-|je6gTey$fFK%c00wGC@VqciQZt3y%|d zFR)b`BP)K#L<=iPgvsa+G`VFWCEqH$Iq6o=jcI(F7xIp4wl;I8&WlBQ{{Gd)%eSxQoAi(-lOvmG9fv#EHJUqzd7a*h zt*ppB#X5_o^W6>4pOaTYRnTIGmaWeW%4tAK;`-o`S7vj=66Gi8hIeRd*jl&->M%NI`;Er?Merl)eSa)ocqjeOW8#Ki2=Ug16FCK%!*dUPb%MO90>4p#F zM}HP1a#$q;xvKZCmZ|Qmk*S1c?DCr^t;{^zRLri;k6wS~0X3wrNBEx|@(>}@en-L7 zAH1m%d03Baiw|YmD_XL98qc%M^LG1Sbu>*2rj(_i21|lVytuj~e?(A+_bD%AiQ?x= zxuHB5sL<}A+j!9~%4e`Ki@9=ehyk>eg;(OY(t4}WCx61$&^sd0m(v=0yvS5bOrBlN zrWj!4jy|NA;{~SUA{mHcLUC{cda*2~g%4MCegKM9Vax0rx#G4Fm_{GmSjvO-XeT-7 zi5?EWW0-icx5Czuf1Wo~IX*6}yvUIN;Z%e=}$L1^AKAMAg?Ote#QCC>N0 z8{0M~wr$%^CKKDXlZl<|*tYG7ZQJIKaq9c|os0iX*R%Rr)zueoy|=^%;SbKX2X2l+ zEMB+FM~)4oEcD{b#c`(^ zZpO3XKtA%Pa7M}NO$khJ*v$DxuP&8q#?0efApWs*bGLqNfxDOzK%23|PZ|&Ri>*h_DikA$q?isZKB*&HEJ!LH7@$O zY&#cr=GEle`4)dEE{%J|0YGNed@_N2MhEKYbA>DmU89#b#&7L`V8Y=J(Wdnr z>~j#|u!UPlKy7I#@@$S+pi={f$zi!)3srW`=InRl?MIp8J1`cG9Y!C+HEJk{LMfhW zDUgU+j)zGpxSyE{B?0m!D^uPtUPP)FgoqG!tY>&1TysvCt`U?VClfc77sp<=HR0&x zV3Sen^F3r=8OP?bhGJx@Si&(ioj?17#OvVzkTV zb+OmEsKPlAH=8r#s4RhS$oViK?Ad~Wi+49*B>DPy@}33!HTPI>Vi^8xcbvE!k0ivQ zBA_itzmli3*U<~;Mw$Be8|_KW*10zbhG|>zR07yeJ;Aw@Wz+{PviGu z%Th1KG>}<>XVh5Bj|!s;w7ppme$udlbD^!d+*5HSH#$xEc6D))aMqneq|*tq51hjY zF9TIGVmB-J)it8O1Gq^)Vdx$kX;)mt&_R)J{zIEC8Wwf^k0xs*!)H3Fa_<-`avvt0 zXe?;$LSraHN0?1a(}`v%w_>GotWC5*m2E*na@Y?a)z5~B6odyLRX z0iUMJBxB_5P4auJDQ3p3f!D(L`I-Zdx7Q-t1@Zn<$+K_9XZD_S?z*T0O&3Hm{u)QC zL!#veu%1=2KZTGV(W!X^re8^4GZ_X`0b$5U?6BCSh4D#%uECtI{`gf?FD*!Zd(`rK zErj6OW||U-M;)ZzCv*NEq0}q;JV^>`kP+W|?Raoqc;g8Y-{#05(T}nrE6Wjg_lgwc zrGG-vPyFR2FVLc{>3@)sE}%{9N_oPhAN0od3yix6^lmhZYt`l9D--sMDFw$4phc+YZ{82xultcSJ}He#-NjaG zSX>k9J}{mcrNONomLzJPp$S>ZUt(tGN)O+w+dyWjjpIIzKrkO}{Y-&E>MgBbXWCal z{8qYwqGE?tJqNGlt7b-lDX*!^;T-6L*su;@(~&m z_Z0=1v1wOG+Eu#}H)Q)+?~kqy1e9|(o-H(gtjxz-4Xh@?lU9<-iwY8x%_Dp}{{n~0 znwGI8w*Ctd?0kf^zyQAj(a(~Y6JXoev}izXEFk2Qi4H2b|CWilipexo5&Ugd-+HEN zCQ2<1ijyy|OW7p`fnda@nZJT8lp_@K*SmFm@*4N8$aN>0(oq*dYTR&3B{#^p{llI% z*?K56U8pPRMvAw%W7DQL2^W)YF7I)3*-O>-Q8hO`gZIbGzblhMwLB?`b@zk#$D=Ej{#xT&J&n4Ye9I_s2zA#V(E91;|O+u|=G$FUT@RnxCvMIDVY0FPNC~ z%`$#$h%gVhg zKlrk~m_MYE5a{B(c8N}c8E(+KVW40*C?rhup}H{ z0$`D38TP|D2=`}?z?Om8!DDK)Fs+=-4O-PgU+#2P1uSlNclf_ACk*lbrNn2HWCYnB zMs5$ZfMx*EL6=2l8qtF=lvcrFwb8H?*MXqPkGzCj@h8xok*r*hFSEqOnw1V2B(7Q+ zBGpTThLUT-!mM&>C+N?V$U}ugl!GrF9QX-G2bWqsT3FH#L}Kc70?OyvtvT}e#tvzD zLTPs73h|h4zPxm%Lw;G;wYXML>AF`CYsG+rNHf0$-B?@= z=dS1?C|BIRsM?HS-76DN;hZ*AE^Wd6NZEFd%~qcJzQgu6_PX6nVP7*H5? z#h|MLMu7h4G9tEjU2CbBqZTb_CY5a*hJ*M=8U5^$X3;%kW4MWtXb__}ZMT%zT4HHZ zzBm2eu&Go5Aar@Tzu6yn;Veh(EP3-U^BAu129Is)A#e4jsyPu2Pua~8#h+^H{8fO~iuhYHt1%xD~@rc?w z-!m=x)M0?FT&^G=c{H5Rhfy33d!5z4fbUgb4~O9VpBEh)SbbETpt1!B8BrH4g_cPy zu$N%FIk2uL(kCQf4H2WN26H7@!=3>Dg$K5$MvssI6oR3G|9*=| zwNk*q(>5bcXZcIE1b2aBdak=*xX;=f)z=&2Nw=Y_vE0{0JFN(Gm)T|<`PJJyBh2Ss zcxxB`srqCS_|&JLLRz)E@Nk-P%tOn%V5+oEqR6AKX}W>@3IMTRHoPpWNz{-xZ6=Z? zJh>r>XRp?`+^5=tt;_7jSm;e&nuSiMVINF2Pej^APLHta62pE>ro+6O1lOQmJ`s@t zT`d`LiB>0DusGB_NOH{LSw+uaA!~yO{w6&P>_x4{Dv@mdJSo||wAv}7x?+UKRu~bn z7j-JTyKsEtJF17b^F%(fk`rHdsZ{W*wPT?B3yj z@H`+!e|z+Y#56N@mR4$P^)}nOv#C5kNqIb$dW5bx#Ate{rNwTX9&hFi{wf>cbj8WYn#V$&qaSO`*m z^^)oe3*{(iAzla_a&sM-4aTj;#^sJ?>FP0=UOiELajC#t8DiC(Oq}& z=;)$&mlyi)Ozg2si#nK%6M4xu5h%GioDA!58z}ofxruSp;eC52&BgEx$zL772(B&a zYA;Ppb{j>4jA30Y<<#zbl}~i+D%zs3pA{yU6%@bU*r`ifizhXfIBcS7?9>Zam2*>_ zo}^9Iq6atM1}v{?{RQLq>)F=tm|eff zTh87~IjRBVA}i86oueE-R_K|%p4Io-%a5)*{(ym7rSAY2>YrDKAIZ4gCh@3H3wi;; zuR5M--ISuq|m$eq83m+!xp(s^d zfgqrZwE+b+KbnrGf`Yq8KB@5mf9+@9L$Dlmjzlge<*AHU9ILWqS$QM1T{cwRhiDXd{Srr!c>JtCc*-_ zh&j@`Utq1_RW_d8gQ03B_Mk@M!=m2<4MLx*1_>^*WkaI}IpExY3h51S^HU@DiU^MQ zxZ#nzB%i~Ff;3QAEE2T`p+vW0?4a&JDQ%6-*R}Pm5C%1dv%a4lhu=FwVTp-a%GQT_ z2)5UL4t{QK_Z$PCe&G%R9TckPv<-UG5u=PLik_SN{pDhP7)x`Gc=-TUa>?`N+K8IFK;2n!lG*3c zkI)U~&y#s74{%QDD&bd15H9nWU9x^IeTl_10qS3m?j!WJ~84SW}NFJ6~A;h6tDceQvjb&nO@uCGQLQJeS;Mr`kC?@u?T;yA(pM?VAN8orTn z4u$HO_5t4lCT|giz@hz~FaMmI^3Qy5GkFaEXoDV$KZ@J&1{{X|mhODOGQl9~Ey+?h_(^6N4Z3N@d@AepKRdSK z96?T+5P(}6CyfnLBXy%01v9yW6`r?ZDy6UbbUPdU8!VT*4q<{*R3-scvuccE32iA% zM07jknLY`ru-d^4D{o-!DF!Kg~+W*<4y>s6I*olP1D{X*>C!{Vk^^M{*UW z9N(F*zUE|@0gJ}3G6^dh(@P{%eYX_^yeK&Y3RJF>eYI2XkUd_4isF2ptTKIy%x`-O z$jM&{7Jr7ji`G>PKbOLzx#X)S-0TeO;|E}PmgUFcNA$_#*GU%DbUJj#W_gW31qIf+ed@!Mu*2w^RRR&o zTFAD^3V6cg9R~2Z8b*@L>oeHbYx}PuEhPQqS|bjT>l&)hF8!wfY@Dae9k<*qJRqFk z>?M~i>qS1&{P15Q`I7!vu1@h)gI-HQMKyFW-i;DVVZfW+s(T>GOwbRQO74$|6?#|K z^02s6IcgW$S+S(N^}kugSZz@lMQH1nmF~Wz3SM`nU>DAsd`vMFGiqq)PQGxaj0Bpl zvmDjY=BFQRKTgdSE88x2;$D5&bANweB83KZQwv=Wj0g)k z_F+U|5YEz*DI~)?o zwBrO@jg}P4n<~SXpPNQ1ypy7AR|Wcjddw(K(D~ggCub-UqF1uq5c|C)a9HJ5F$0s= z#B))}es0rc*@j-$XjBbRF^%ete0ax*ud?cW0RAU-_H#-lh?CKcNW0vYh%Wi<9Jg7b zS~i?UPZ#E}#12HS7G9^u(NMn!3HRbhotVE>SY8B{e^7znIn1sqRwdM_jf9F(djm#d z94sswW082$^sDms`H$0{;!*|Q49FOVVa+wqV(>C9ERxGRT>^sUw>7QwV7ahc4;C`I ztfnoQdmsr}#*(5oF&CmcVq6l{5kx{t`Y0UZy8H>DSNwN7NujeVIWr4`)w;uwRIu`Z zf4=Dwfia!W3WY^@B@5>Z5zADG(S<~go7)y4h^+2egr1=KDqF8D&AQ0hK9_r1W=8Pg zG{r|TB^V{OF&>!dD3dnlnpE_^t-m`mFN#=m)No|?GuZEb8yj0VLvG>Ttl>V0>Lb+= zY#yLv!K7KVv1hd+888jho+iwoQtG67eD2Af19@OAbv5Z$RHG)I!dkM*T?d*nBg?d9ZaZQD^t*wgoM6)uv_ zho&FTEm{#2Z$zw|SrI7K))E=0Ow;`Tjv1D5;ba8%abeS>pm5@Nonx5(56(x@kzkGq zz_Y*x?&b;`%9^dY&X7E3JDu)b1161qKf8tgBP1rBWbRC^pciXs4a;)y6%J!&C^^)S zd9DKME6$DUSSO*&&d}N7DK7Weq^V_{3nM!Lb}t{b{!Ns2U0~$;W_{9JqmlgyzPma6 zAt}#rcRz^y<*{re#zj%twMz{FxX~DGM zMw?q8TQh{Z_>5OwoTl|(v2=q;;2p`I1+Lp6L`HI4Z9V*|M)_MoUORUx+~XS3dczf~ z;NKHaScOU`eoEN4zb|sm6{juj0;kLpIMqfF1T9DiyY}Q!L zpY~bKpMqGJETyy*%Y~)!J@RJcRaFP))I`Uvfy9PXp_Um$Z0WrXhV%k+>a`gN`A22&2hO0h5B^pIHlJogBL4p7 z+&hB_c@XE7W!FkWqGyHFQ1&wol2vt3OmqC$n$gqoE=zi>zOS7=eCE`1J6P99Vfec3 zuYYlGK)?))b!J312gCEizS*+wBd+R^W4s>XQ4sH^zZLpv4IW`U@fMw!HWcN*3m zPKjr29vD#v5>aaRegE66`z@z~{rzmc?@Y~? zTP*PmMRKk(th=--*ax>G8Oc#u(4&OGE?J_g^qi=t)!U*0fc^ILZf)|QJhn^Kvvsml z$rskSns%R5$MZteovn|IKDVnCg5{@D1b}Tn4&fWFe+!qB5et`M9YF-<`IooR%}9Zm z3+jLt7-e?h!zW#=U332xpfa9h?TB5^EA)mGp+k}0XXKLS{ey=h7r29w)MLn|Y&+0q_!-tN`SZ02w@ zR%B$QIJC5f*F3;`J?rxBGcSbYwa=9$4jW|c-h!Re=`GD=SW=#8AF>E$>@VNUzAd?v z=c!%Tk4giVu$$sP3~Noa^~(kd;l>y4;ikGnKb+VEO%=XJtdJj{)??2M9m@s0N8bbt`3ba}Kwr*t9XzPq<3K zSYm7)wI{UaqOPZ6cX(oqZ^Q8F9c$WcsYwn{RawX@_p z%l|K|oYks`<^V;)?;DQf z&uizCxjTeq_-)r1kI>3-IZ-{}0@YjrEGGpEOYUzj+5QV~k(@NR)(lnj%Sn-w2h!t_ z9Hrr4sh38)Xq+H_lV%B0c;&g%tq77hDnmJf0}QYtU8F{nY3DGkqV+P zQUkv&Q~}+Q()pQ5MX9S>bW1OPuJfOkPci!-(_nJ&miXkGwR?jQ5{dn^EW$hzzf$SJ{fs zcz|{BRyTT$qM(pmUJmS&^{xgWg&eeh(2Oh`*!f1xdn0})qEA~k)!ZBCU10wb z=-R@c_VAdJEnF*JYq&#q!^7f@ktU$(y^5j~Su-%6)G2O9fw&Tngkjk?6v%z)B3Xg~ z1?3!%gfZhzb(CU;;m5a;B~EPv;_&FjC)c$o)vi!6guE&#4=& z3;2ciAXS#@<^#FMc!1|}kdOqbc}~+L{l#DUa(SH~bKo7xL_4V%`;x4Nn?n-3%5auX zgCE;cGoF#c@h3}>?q~T$lp9He=Tni- z@K;RTvHcfHssmQ~rtfD6WH6BN^}|B4-0wx9h4FA=(1;c&N98dE)PguU3`f^DBy)iF z#~Hwb$e16?>44{hNV&lM$$<8QQCK_O3cui^1&SaYAB>5Mo2d zMy%NtpqA5VG4ka08S4g|_{kx_s?R1Hf9g06`(?&nZD!hwr`LWLc% z)NkL2_P#N<=gy049^O=5vJaK)*L;$$o##QXY&QXm+~G5n+@Sf0@P`0OZcsqxt-OLG zROa4O&c+gK*-vq)BDWeASyoNKHk9B_9FDrtVI*RyZ>qXPAi||SQ-Zh1n}gneuU54xS>Y|GU_Mp!#3yMB(N&v04A$ zI`97f9{ul&73hk48F@xj*^;$N4t&{jWCD^rg@lSAg}`BimZ9?;Z;)rj_&ihrC$K;G z+s=}}f3P4mUB@vOk?31Gn+z7VO4%TD=H64~x!j=TwR$s4=oS}|Y`}lGIBH7S75xUc zg{5E~YC&E9Ljc^CM0`cR!4?2+3z1(Cvwaiy|I81-`@pvsk+9Sr$xxpwqJnB>37v|$ zG?&}-B9cvI{=eDcB2q2>KN^i2H$#DTcE^{0v3gk zu$ND|MA4KUGOe(!dgp0cGl$P4Y=oc}6^YJ;sa~mV_@wT?uqD4k+egfY?BAzpBvGnJ z_kwf^#0`J{$Q?b%8p*|i==2{)VB=Zf7V!MUVbTJAEo@*vcK0mBjvMM5kX4k=GI+Al z4ab1X7n^IiIO`;Z!YORT-{cZm$@;mz(2q?P)^gQMw9voCvuC3%>R!)>{otvbjHNeP z?jSvgt$hLu_hC4bkUb4#dyV#hy2``oUmeI6M?=B$!I)eRX_2e^Cd9%0W+H;ry`3NL zL!Z=lf2u3!X|yNA$It6Wme%NwJV!Vo+Jg0)ivJwPA@a>=-yXS-aaU16cGbnJ{oMYV zAm$yB#jzsZvg$O!iv5R?X28dg9=0_F>!l}A`pAe=kQw>7G}vjT0u+>_DR}3XWCFhC zjOaEgTLOo~d8iyHiXZ8ZT^boD^#z24>XuV@(oZYom%PX%1|L*&CiyhCoZJKanWFU= zHQ5;arxG|TU4&B}4F>}XpD$0c(s%i(u&u0ht2Ck*s-|2SGJQ;MVz!(UM`)5Ul>!1j z=X@Alni*quUThMG+jNaXDK4(LR^{L`N}^h)wc#@o$&0kw6Ie|`%)ehz&Bj$LBJ>0dpl>rn-)kdCB zoRv`-TD#vZCd)2JG}6N{8aL+dFM^zu+`skB-31nXJ{8A{<$K!3@n%Mq3_or%ZPR6b zGMzO(9WcBfs|0-hE9AbxO1`Fozr{!U_u=QV4JPL`DRH;M9(nKcvhfjaXC4hIMGZ)0 zhoh~7+!(PE!*DWfu`@Ak=`k^Fna2E1Z`@EtLv=xKwaoUht7D%5CA89?#j)CwJ`^=6 zb`le6pzs)hAAPq{fPxsP2n$YudgMN6a7eI*>-)mFl^KzQrGWsw9fm*jcNtg>KJvBO za>ikh)qv6`4K{_4W0hcAOzcj{1UM+vXyL?n>Y=R3^=ejn68W+3cG>*9zC63Uyj*{* zUuJcM7Zq<|fzM03(avs*bwqA0*Gzz#o>l+61RRB&s{d(OSQEJ|q%tF_1A7N{fC46_ z7-48-JKz#pzj1oo$6e<2D&d=^HzoRcFm*nx(I(9^^@hVL4SZ@&0%%b?=-RJ5COF3E z%&Z58|H&UisuqsglOt&4;&ee(x6DcG&HK3ic_upbwK)niBefuE|pp`q_Lscjfjt zJD&R+toD)?eI@Yq4Tqv5H+E(WI?^`bL7eu)4l}r-?g+K7 zVna`T7xGfJYxw2t?e6aH2At$N75a3(f9Ux(#cih%a|emNkhH8Q>TD2@bQ>b>V{-$y z93smX%QuLG-f_ZEBDaQfodt7wD;($nPEihAwOI0_o8sQh-O`^Cb|0uYfkO{wR$yxo~}7L%5Zcc z)gM+pHhG+`5^3GUHwljzaW&dH#L)Ygt{1R0UKQlc>0&YY#DXd~|HGC82e5`qKx%ldS@So|) zEBW#9dK#XmLe`kqpqv$+j@zAH)f($QpobQls+u=)_e62wUt|_Jr(#S~u|r6Y5Ys~! z!Op^oB8k*S4@$z@`;Fcht8j?O2&IVQzR0@gZGR1&hjMY#^tvdfIZ4+38;ZL*EIR?nUJI? zl*Z6`RjpY=sXRBCKWlA=FO1r|rfV)a!Z zX+{MWCeBa_mPAxQ8d2Ih5jP`0!p@IGGL7SXu!sgMMJ6dFSbuvC%=jbxZQ4*3R?zP9 z>3$hJAE2_}#}BoWFfADOPyPE{TInd%p0IJ6Xfy1)) zu8(!8zr&j;qEz4C=0gG}kCk9FN=~Ch?k4X*@=DV3k;?JZ=VwjJ*BtB|n=k-Dv92yj z>*XKSG^-9@z%9dl>iuS$4H>IGMhZjUl&ACJAZTkPZcDG3T3WH7g9LhZLFg$YXGz?K z&5D*%X6I)q8KW5yx`Wb}uBm6FW_#=dOWc?-C_xzf?(dJ;ACcLzyf44^rPg03z|Z+D z`urZvW*w{)+vdb$Ct^8V$B+3`=90Pr?_%42A^tg5wjZmNfD{w)sMbtZz z-)<9e*iIB&ztv&yn@+SiHe$(;+H74;rW z#Z2{4l{UF@QS?io;&jRd0XnZ`o~x%;6^(r6{%KE`m(3TmW3Uw+yG1+{lUK*YiA>hr z*c7Z|*a(PVDe9PXmsF9QhHx#+)9DDTTkABOR=8g<7dyZ%Nh17~;>6JnmWQclv4 z2CHIn_ZZjW`_!Jr!~_H_IRO^ct-QX`;0#d)HxoIv>^#E2YeMIwlV1ZX7)un+Az!0o zMw5hg3?s!!hnzC57rSc@IsnoxGcA zf9sW&^=RCCOMlh&^HjHAIWBW@{$yQT1hxh(s-w^-*jOx5!rNyi zTT*_HCTCC0Pm??1#pHTeOA5j}B#1G#VC19ReDs*&N~;t=e+l(hA_^#o=r zh}RAi=M8cTl78frts|0H|T*ax%KTML)a)HD56V7z7J=s zZ0D9_%iYK$9X6(X&2}Xw!m4+1$t=Yq^WYLB1zrV$$^ApL0d=P5QcWbdmw=c4niElB z$q&h#m7nguU&%)PMb`SEL|DDVxtfMVMrpa1AfK)liWFWF7g+VRSFZ3mOg)OLK+R+} z{K6vk-p7a?;ebC$?`k?g+r@K`XL3 zJVFpX-)cAHRg8_y5djt5ELiDjW@)%V5KeymLeeB=gN2-eVu^S=v?$ZKoK=)29HoGshoZUwaN#Wn4xfUx^m*V2a&ST=lg}{>W0Tz%C90^~Y zgejhD&}BRH+~Ol;g;qg|CO2D_DTs9h`OfBTc=+h^r4;gvF2Wk8Mz+y@kvH$tG8i)% zSxh5S?^$Q@p8oM|ZBg8VwGY(X>)IjkH|pjs=2!^ISTu)Xd)TA~ds`A)$sX)MNbY(y`n z^qZQibdCD`pj!8wtn>#mxY*2iyl^O0iQtk0qZ7SW^X9L-D@NGc$sNb zuJ6Iuxy+9(`~WazN`k#=vYZUe;w%!`ENTYWCUZfkPo>=66jL2J*R>yajRV{Gk8Ha6 z24uCT6$Ju0YD=Xe5S_WU^mn;~PH`s2sICP@8rQ==#{ror%v?_o;6Lf?U`>^rTvi!w!75zL^fgM~?!8mAHX4>=~mND6=6D{$6 zBI0t7KlSIE103?`1w=ri{e$9_VJK|lY->V((oPK8$O*$nI~Bw|v%T{}w0q3!OsWI* z;gPiH)9>c*>+bb*bNaStvPZ9{N4n2KDA)`-UQ+Kt5udza-Qli)nUuWRxE+@V`y&=?0WSwD9D^k!FoNbzfru)~-~$WdHZ__C z0O@=^l2M{$#`C=Y4V#PX)P1k|s|C3nF$>wW-4en!AtRt_hGCEl@xc#mAz#{gHH*Ux(kBass zzM3CuT-Fan`V5P=;TP+WOVT(oGSc_`^c(UIKlf_~Tf{ms@I&}G_NxOFf`3kJM6GaK z3_;#=U_zyA(W9%woM2Mtt2e#N}v@a`0nZ*ab;D- zP{p|xUnAOCdajjdLrVJwrLy_2;ILsVr&W&Vq-PzGd=Wu$zHQvLkW2$vhbAPYs#8$Lv?MBwu0Z%#x$qL_$%u72-$le`lwYJYMz9%$x9n5iObYic zF0GvPEV6Qky_7v3rSIn8&n6uX9C*5AVW-28H##(By4+`7t+6C$(C>D1O5)1p>-(QA z46Wn$FGu9KYK?p&quEIYhvI*e-h^LrqQ1_|@Zm(TnyFS`t|7*~HArbp7wM@nzLxUg z2K%F=9no03tcA14_t^%U3DY0ptX@bM$?&#Vg&M-n<}u?W7 zFAOV8{8Rj-mrj0{G$Y8mLK|qRuEEX4$L%3Kqo85!9k;{Kzo61}^O+8AM`p`R)!{tE zRp5;!n_diN7gJBs35WBvi`ro&;VN5a*cmf%m01xdD2g7Y&|)J8x+Pl(wU2-Zy9WKX zL=>kjzC$Z^ITZ{RG!ooXY!|k$udu~*Adb-w0F%z7%?#N6fMM+<8K*pFjGjNLjvS{@ zbt8%$l1s0#Z8Goy!d)C|)7}(Va^M$4PlAT`h-8UalOEyV|w zON}nbYK9re-r?G04gPBy-ZDTkDdY)9WqDVqnkbXZ%TK_LXp$v~fF`J_SnHLcxH)^a)!o2(hh2HWt&KC$Hd#@URN&9vn+=6iC@3)YiF7n zD?NuY2o#{;atou^F~0ic1Ri;EEwX6qG|<9w&t?T+M~^lkX*7Hl9N{vCS|`NWiNp(I zUt;$|J5r)fEzQc-5BtO5boB9APIyp0z@vF6Fa_s7Nqm%$AN%ZXbp9G9lNDIztZ~(k( zJ%pCDu<@or&Ql{lI~nzinpP$F1q^`WpAin8qLPY!W4zFN8*ZS$j_qMiDD{=PjUb-T zqe$*BDL=fZTOk;$o(UG{Z+qo7J9@w;m_f!xqyryZ%~^$*fNjxww5SxYZ2x0a zbAdYblu-On5XtPt~Ho_1YBb2Pkf!p^zNs6C7b$k;-xklgols-^PSO3!OrrJbDajX)2&(6wpT zDxfM)_9TH3LrDHCfnI^(q@Bn|cE<(eSr+r3y~=pXkEyPk3g!eLYxpta1^%c&>jCPp zs1CZWRf^PrSW|(nIt-eQKo>;F&d+^Tz;>kcM#UKnCKN##s)>V;&le2hZ<5e5c`dsG z<0nh7#Y_N3PZ^AsA8evYP;5lWXJDqiU=DfW8@v>8qVcFEcmKbL@?2DTaqX=;>V!S; zJq6;ZwYgL&;reC`;iuY8v`jo`riX=q=cw*yrW$`Su@uKHfcRwB#|=6{%TfTWC(!1|`{w68cw)!T%*=^!S2yvEkdzYir; zh+rQi@Di7A9u01w4(I5SJS>=R?MoV+2#R-CnFr&mq$$bLz~j$_K|iGSYny*gA~0hR zqozIGIW(lHPh@KDQEZq5t=Q3W9%Na*7cGd;eK2W$3rv1hbfOJbJ_&yC6lMkApKyxFU67KcLAX7wq#LB+VO@N0J$LxrgV;FH$^L z-sV}!Fk$q%hJi2y%0P7a5T4;6>3^0J;U!nFYR$}2f&gEZ6ThP%UV3)3%!}*aiyJ8q zkGKu#Fc*VDP2yDFTB`c%NhwDou>+(el0|Rf`>=^91GB9_G>B23=@+y8_|pUZv|#2H z;oXolgc6oPDwjV&0D*@@HW~b;5i^Dqsa>x;iIT1)Nh)1*7?M?Xo4!o{M&4!v5;ub> zjQWb6nzZk$&|z3)dQ1;veEw(@6JHqi8BA3tGH{Vp);e={NGB3GLkgy+u!gdQ!$~ld z@~R#HY+Uo#z<-v?lwy!JWp`W+7DpxZWAGJnG#2q33ROV@UE(iezPzk!qVmacE;NK4 zvrm4tEZv;=lO?&v>=ZU?tB`FM*Up zj?qGK(#B1uKnWrLzTtbg9FHdNx76Oh%%b6yzElR>?79p6z>mn<;-IbEz_9h?s;=Mv zLu60i#h@4Oef_qWsTokQlrcD1fDEYom8xrHl^0f8c-1FKuytnJh&U02i>mT9+zySy zgR97y!j8>L*%XchCps&sVgvio^7m@W^{3;MaD`tO;^$>-7~*49WJy|x{zC;@n{Gmr zsA%#7X2_FS9B;J5(%kh-2Wk}}%t>Ppn<7)c$C`V7@^==;tDXbTBtPD;sAyq7Eg}H6 z;2Q#iEFgs&U*^i&9r(lhdR^Z3*sp=&bzN*&X|UP)4=d0b6T8%qqi6kJ?PdA*#;fFF zI=iwhqq67yz}>P+rJm>STo<;idmde5zTPS+G9EWF!AO9{03pbdkV8&skABSWyu`ef ze#ZjAV=7y%6_GdJ_yvC5hq#rd>NFCb1&iQ`;GdA1f=Z9TJi#Ye6JbPbCi$8`Qr5?X z$bk8FJOF;7MI!6L`nXU^wRuP1CWy;}w@bbiG5R+@G&E~({sy+vXV9u3g?yc0@S?Kt z%t!632_t@o8?wJtyq71_Q3q<3l&QlQ%TPlZ_*>615WK1@B{=662#Y zQc!u&4HQz@S2&MRI$>yz{(mZFeUpYOdM(CcaBeT6Jz3ER{wyQYQQ)r7ef&fEA-iC@ zq7kLB7}5E1R0z43^hnxXh{PAov3#C@p#>rm5xTo+u%LakzE!i|z@lIU3X-O9p}_+4 z-$f?^SV`qH4zUcaB3PtNZMi+jDr$;|2;bTsB8SR+h-&c#wC4-1@s6(E<|a-byV5j#0r`+{g;e; zBdhpgR})xNdnZ5-IQlrHJShF5nQ~-&Y+8l?kA?Z@iR7L%t%~o8-9soHO&$rOl+5xp z74s=d5T$SWwm+f}N`vFY8;<<#znho~mZn2<1%Luu@<80UWqwr#@fyW&qq21*NrV_W zDg`*9DH{2;iM?&dSDXL#1=^jW<)wB%1hO{x9emna=LbItv132Q@hs)zW{%@exm$WY zmB-6(GS89VG>sGPN9s*0(aj)Ascop{&~w>mwz!K!9j+kYM&GV~OvH=|QaMcb*up-v z2-e@>pCFwtYkdDc3`Xj*ljB0;UNU^k(@pLEJVr#H?x?UK28UJ$(bOc<8L_8aC*uCCHnA1 z*ooN_RlI2ZH*d_4Z#J@IZa2+quLQletx^dip#CUbvIc{qUKG`GOFgMCv@plPA4^2+ zIZ<#k1Z>Fyj8m3(Vaq@uK_ZRHSO}WZ6r8!V#tpzh_6AqPp9nem%1*7@8fNXOl;hX5 z49d8$<~H?R&F`{L6+dO2k#(p{zSd0@&8ric3- z2&71`-=^X{VA@Q8u&d)7r{3#nFe@D2I0B=&$dlrg_&gK_5dk*l8|%1ZZkt{dYOk+u zx7E*Zm~!me*>@kV{r<28^b9a)UGp5m6G++9-SB?pUJ$3m&#;Jg48Qa0iQumEe~I?D zZ2eu^UmT(y_=>hi85Y^pdA{9D85UpJ|H58qxS@cg^dsWFNr$F0SV4qKx2!BUpLP`J zacQ2&1tMDL=#P5I8ln11bq>DhWOgN50nrQb zagb#cMJcv&gopOO$mq$*$6}Ncn~F?82oL}D0K7>(bT=>jo^H)fPQk23lClFqp-~V6Ef<#A$v^R7>Rf2)&&A@~rEM#aQJV>qk4+I3LUFj| zAe*O#*T+o$x0;n6!}sRiYW89cys&iWj&>I_meMr#y!N-RhgGajq~qnE7=_ceWmiA^ z+^OSTC>d$b4uSO{3`JdX;yLwq-&iYdrWatJ?hIO3)w57^_l8yYrGc-Q!7YPq96To)K7U)%}srq+{#r_=b_1;?g>mp_R zh5_BswINRE7w_O*k?!gm_0J^6(xJbOEeRW3?U=j&Z>&)PP}yzVUEX$<^#UU8^UJ% zpxE?qSvEDSMi8bed);wu@WiCp2w+(R_Q$8eIvByam%uViBoXwvnHT)`9v&F1ToD*3 z_~UVJAD-mUhsoRE2ZT2c-d8UyDkRi@4Nq>o)LdG=nI7D?;g#5pxEBccZwcA=5drp& z-M6GR8V|3QR5&!Nw@gk9-;YqPFMd-(Ij`6f{xE^igEqsF8i=%*t+zFDU09aoCZsdp zsU2EaCS5FL>YN?nv@sJb5HpjJAtwi zm0-3)4A9by(3kAU0iJo?s0)^b;f9Y}ep{gCx)eQ5$FJE@6nV|#^o2T~hUzK~sjg~V z(mX~!Vq)yyab9Xnwv^oj&r_07P%co@CrXXNf1dCWzHx#{T<8y~nFTIq#t?QjI+C%k zk@xF*-}!@*_Nan0n`V$R=c1%+A1kcIjwK;DUkwR3=w9}#<5n)d7?mUE%8>L%C?oh; z-c~+H5A}7}kC8$=8hh)LQ!lLY!#QjLV4226L;y3S&EPbn{<4iNhaJWUf#joU(u#`a zr`+)p*YQMeQoL(sD{YxCA;R5PeAMoqM9yb`xuL8W+(Q6RkOUZGA*%8!;Wh_bp}Xc= zF%}(L1S>q@2JDE_(A3xg-nk&*gVM$(sH9S_G{&H(EvjIbWR83YltwEnrpH=lq=}BD zLUK^RBJD*d?GV7^L=VJfsan4@Yijm=tK~(dCd@Pr3EM!4yzc6qYp~EKK)e?nkRdwF zd9Tb$r+~Yim(3OVeCG!CpW(l|YjkBCh_txn!2x{#`6lIS`@9(Wv&CZvF318M38OzFZe_TGX zY!l{~Yet~#i_^WSw_aocx$K-LK^lhZ*jt8(Z56Mhroq2OFCN| zOL`Z$t|A2|b1LC#-ma75z`PYOa$;%rfI)$Rimib&FRw{W5KomTO7Cl0ljLV=*co5b zc8JykrkO-XEvuZ04(ArvoqCQ^LCguK&6h|$EaiQ=!^Um3p? zS|iYkO*FUeIUr6eqn*eglu7=h>bj5rM_0b@sfpHFj@3OQdB>vME5YMD}r+Mh9f zNgE92p-{9RR)}KL&}K}IJ%N-%q~t%{1hFv$PVyuj%Z?n>=0Qm2!PI5ap5ps#^=H;1YP4PRN(J}&CEbg zk4hETY|7F|M5WZAepqBgcOdu60t_cZiwIoD%s4EPNmDF4tNYN!Zsq6QjxNq%NT?2) zsssm0wrCgs&WQp2WOJicbo}P^Y{-v`bb&W^Pcb$*Mz+F#L^E>X*uHUvtt#0_g^e6C z?(V2AL{BkR9G{4f7HUBm!@OK?C1Fy|LN z4x8_nnAmgq74ujn#_Kv(;5)VWW^nWX17AxBxEJrION%3-oyCZ|&~Xzjeo}~#kj99% z2vUHf$W3`1kE3Ok=y_umOmKc_Y>fu1L402-kNNTT5Bf(afzC~kJS~f|rAE4E7D15L2C8*1P5IQ3Fm0&jSvH*E%G76Es@(#A^G0$F8B#gG6{5Sa z$YVidJuPLWizZ3S*q!&ZOS7%F@_+}((+ErqKp=TO?>F4u4nx!Wn%^`IG{3cazSKxn z;51l_MPB1L>2e{fN~h0`b~OsB-r3qKUUwOH4Kl$lHu;k53h8oNKQVJAOAb%qp{`1t ziNy0*LXkY87ho-nb!nCb)B(neWR)6to@Z468(pnUu zoSGmpZJGttst(RiL9JRQL>s+j7u160d?c{K!XIa^!IbD|;e4=$S4a9Rh9B3eR=I%U zTozM;E+9A!Qrt3_0@?$*bVsZI0HewCm+FWBnm=!r;{>T?Q=y#hb5L~qCO*q-xDxOw2GurX;oejeM128Roaf=i=v@8MtVToOgD^{+LX-AM@4**N!VK zF6x%B{Kt+eD>}TQYq#ie*xDXUh5(!L{(hG$Eh=vO%>H_g`Glrztx|JIu33^tII7n~ z6$`4m__V>ym5_u$;t#0<30l_lm!m3s{^>y=a2u=^Uya`Tn+0>SDneH=m>2fBzz>qW zepdROR_$@tPjvFK(u3w__mFP71U)peF8!kS+Q+o$F4kexBUoUzjep)0VT73yK5V(2P;5(%dmCO)=%GJ=pvO?_RK zeZH8%$aD@7(}WAh5iVLCt`R~C<)WKZV!ADYBe%St-QKFc^=E&H!_DVd?@^btA8i)p6CaBH_N!=z}zr-v6vv>2gAh%>uakvW{y-NgCG<$Apf z8-F%bCVbo+&gOs8^0Eg!S8@Pf0$xY;@@Fl)wnBR>ZqlSfF7waR)9UCtX0Wh5TM3*R ztkaI^UVTeHbdrukj@C%nE3TJ%4{QH-ewBcThu&%=J~{5sc60iBIcOiHVox~&S}$2s z9xt})3Sd!Q2pWHe^pR)hl8O^`gBnVX{ow71J*tGKJE^e`=?*8D!z8!FEd&c(9)a_#al?F4^_r@a>+TB)h>Kohko zpnbV`>ZqyoX3>^=?K`#cecjOPM2Pj7hDKfW9g0z*gL`jYG^_oX`sz;llgNgRlRmnF z)d}W=Y=xoAt)eI;L*fN-*oKYg96Z-?i`#p1y2W4RT}a#upD zwx7?2ACn;MF+`hSIBNtn8t>t2zb%+A^zXO~X2^W9&pv7#H`B;_&sx5@GE4@Wf`)6?ju)sB?Z71*{gx_rc(Y{*huU??+ z60y0(gu+EWZ06fTH;28T&4>Ro%hW5>VLYX4oB^WZA?( zC<=4UgS9bFQf{hxqA9SYZj(qrmE4C_uRiS^92eYDyQp$btm(w3q3GZK@-+~L2MSfW zQkCEx=WttDEr`sarpD5i3_P)dGPTIxqq-{2G3z-q{KiWBjNezbiA~!4>Q6);uw2t- z4-xQ-#N-`DFntQNM5|~BxX6CT#jEQfZDw9y;c7n8y0n?xV_6pzWu(HZR|Av~AA}5? z2w)>TRF7Pkr8%t%5*ZT?D1=Hr_v|))oagZ zE**>#T{@L*k1IW%fNyb5HCm^chi@k)yi%5P(Be1JUWGk$cIDAwte`lM8FddyhbGh) zx1Zpc24KyNpx8w#29jUkrTsCC?UVb^`ix4rh(IW;B8@IS`O=dV#gP6*u4-R`esUARBK@e?zhIM?bc(dGvnQhucE@eC{R0{9Nr~oU-eUaHixJ_2b`I^ z1w*x$52AmMdKLwDI1a5@ChgwWNy&3)Y;NtCrO`}>*wPWj%>}NpzWCk}>TRD5_mRtEN+& zFl)4YSjrvAXE^~0h*>knYcJiI)@GOFYegTi>Q7Es^l1TEKul%T@Za!eWzU(^bdJg5 zLHEM~AVV9fAypKRrd;f>#RyuP5oHmx6Qvq#C6!>v4>+CUNC?j*Dq9`Cd!*Q*6!l>xGZa z3J(0O-zTg$A0+Q+8p0mKY%-k8AIW~&vl#Q@isUqQ0Zk=v6mPaOEJAJ3so4rjR-eMg z#pMqj%6na#EC3+uu|Tw522|g_x|R55K`H`sq;hjXT{2Vhimfm`eDbqQ^X6?etwE%ODVLpkZ`j!s62`1t?iN9Zz2cpZSni^l^zC)pci*G7_eT#sCOz^JoT@)U zhjS#9%(6FYJF52&o~vn$MRD*x&1R4N(DSybVv|#(wgtZ>t=_Qju;Z&WYl}paPP^&-U9juc%a>IlbyxqPhFNu;Q=6pvdjV+8sVFY;SuS=B=;7ad>X-mW7dO z;Ms_U{_17>V8CV?C1E3n(nX43jxk`Osqh23{Nb{|wh)Zg(o!-vO+p>vSza^&l? z&coh4$cH}hvaxC+Q*QEj8(^x_-%;M?>7t`}DwM$1SaS|k|Ez=S7|R5ju)_sJM5#!> zaw~`+JupXG;QOZoP>Sup6mrNEc_uBV<1&qg@Ap)C2wviF#Qy*Tqy`#)nb*B7{@rMB zDgC+hFv0*1u{fl`ooC2eCRyY}32p3k)+<>Cs;}C-V$CjqA(864UbnB)y%Bj|?tVKf zeqfsz_E#4WhItnt8bn1pEVDH15hv4REXW$kM@Yh@NbDj^C{&7Sm$5opjD2+(*gx)M zvs2-FF#(po{(dbmYtFso$1`3#&kTWJ=Skv%_iP)A7eG6HpKF9I0hNGxap9l1iD~Ov z#yXV8479pokaT@%uHz-L&tK0rpsw-qRAeqhPxno6r3KVtz686-Z|qprKZq2^^n4J+ zBzowsrAt3tB?jozK+Dxp6f#I_*rnH=6*`rzsud+b)R#cckATBO4>#AmSrSdZ%$zdH z##Dtp)7UxS9SoPFy0&1>LC`G)rE$)YvMw$z$2yvZC0V_dBsx82qHIuNN+$6fcqm8_ z`2mRU#ag$$t{ylNgu?!XiZ&LSA{ph0_31s@>{mMAV^GiiK*u{{N*pougEfO$?;C*@q3T`N{tXR9~D6>@8u9EJ{thO21DGzJs1d z=x)AF&0&FL(7kkYs7yPGDxVm%9l&o7zko_cvncL?=JxEmJH z+`wSpSq(ThD6J+S9*24H* z*-0VTN0++dV9Kfk=kEPAOV|Dt{ZN-Jm?5E#F5#9D6WKUQkgX2RKLZgE(ojClha1M0 zc~B&5UO3i>Nj8$N@%)jby{NIGWhNLTK;g{Uaq}4&DP)<3hj*uFZZ>MibPH^K$5uTt%CTj=1@D&InW?;=r0ZFAGb=Z9B6U8rE>h=Miv8 zvZ4kA0@zDz#jqPes|mfjO=DMS$uhx7AY65JvJ~frp7Qbv1c2tExpsgNSRKPVRh8eu z#l#%(WdD?j=yyX*W0$(L(2n}#;x_NGpH)X4y==~@;ZRv_ri{vTXI)g9$63RW2B4no zHl)JqzkrAK6(A)T6f|dV5G!H(N+EN1+JW!q$&DMA>x&L7>d*a@R!P)99uk(}k{NTt z7Q)=HMmyOJwtS{2FI6f=vJ3}XRHkB{8e_cIR(dt8n4-R$ zDYUHmykdIsk@UJwS+GuG>nBpioG<#a0E?;a0JUvoD*Z?9G)yI_mJx%~(ZPFikZdeg zgjZDSYRT`6O#SM|JlE9b#lF-rB;`CukySRquyYZ{EvUWD6n1=v4c_sZAxpVyMxux_ z*ZLS!>D}&Jr0&XMz(6CyC>wCRG28eB4JNQVyzOXc{At@Y|I%UM{`RJ+1!=wbyjb*# zyvD9Sv$dXZ4vM)>T{OLH%QBkd&WfamBF;PbdZ2#Kckmj?!Zb~&na?+vem9tBt(R;0 zw~O%~00Mn?5EMWX9(qZ#hX6`Lb26_pL2O2cO3VUAxgrmN7)Bjie*+3FKY+= z#eeqCsKAz9U4U0tq#WRiOWCrE!!_MEfIODg_%w@o=KpfoG%nIbapZda$%lPWHiEj= z*)*Rn9T`DH!&h2Q#p{>=Z2S6LQy^(NTen*E&$QCopEt_|#;UVdjV056t3W~vu3T%W z6rQje^Em*(^zX7Q!UTW4^kPfrJn!BpIK8Y{^Dj1>pjkla*_F7GmRE&|#q1fzBr!y- zJBUSp$gsw)D0sI7!LFT~?!joBqp*H#y_YnhF(zzvImt{Y*Wvw8mcNYW0qD9#In&Y* z50_BI!230FG{P)v?+Zl)OG6jGXO?d6UpB(YbmOWQvZsK7K8VSgH?aWdT#zx>RFYs= zG?g2T=W8x0GRF`k!06&z~#MgP$ep_vuBU@%;30`up-94Wl{2#}XgShg{R zyC8P>&BtvQ!E|8suO0=Gi*)!KNM6y8(73}!yV+a;XHqrqB(CtRh!#hweokm!#2a1X(q zHy4LDgc44Aj-xrs6a$~jH3T>_(Is0j#(A(T?K!6ZDv@ur;^Z@*p?W7FiJ^$AY(?5n z57UdtZ_Wx$a~l)SaQ3i~$~PmTDB(N+zPDa^qtdjpafS1~mAG~mKA_P&bL{|@q^fSS zSV%A*seF~!d~m5t<1)A+FFr&jLOri#x+_B@SV3^kM6HUE3e9HNhl5mSreMcH$Kipv z$*d-lQeRJw(YVnqy+clStQ+T#0XLh$BIc*YD;FzI&y491Fc#CE`SEHqqIY!;qw$g>S#Kv%zeuW9R zR$&HDsB86NLm7`80zI;Dop0}*^4M1`Tec6lR z;g?-VRI2p2wxcOP#vR{m&gKWwgq#td_`wS09oOn9tY%)=jo2p9IJ)WESHHb^-o=|e zvL{T|8C7n(_kw(vRL$0MRV%(X+l$TR1fRaXxUdXDds5H_ECxjhyfL1QiifE&EyybT zd)o5Q46JTXzk}&lBA6U;h1jGO#U)4bW_Xfk5H#k2@xY-FhOMA21`j}KDNwcCtTdr% z-=P%lIbrQSSp|@K|mtQ*|xS zNLIK3=SSa|0?hfW5jpq0Hx&om5Z~TJR|FKBzDbKU_lFl8{Fb=_gxcrS&5szYM>po$ zz7xDPhRuj#Xi^m{UQ(o0TnuU);wmC26^a)=WCUM07(!TUYUIaviZDKm-eH z0W^B3I(|oAnk0g?9^zi3rJetd5BmrS`X1a=EkHCiCBpCl5=7ZBfaR8-ikZsyz3W+r zzLj2?|p$bbIOzF_%%!!IG<%601%AMp^dpe))=hLH(L=xh$?mywgODA2Y$$0T>@euA6f#%;szSF?pS48no|`XSL(Xm0d7sA=w8DmIQV zs8?~WbTUU0b$l+mM+RiA$R9+a!TC~2udYBQCPgRAy+1?yd<%T`a!*I ze}v{yLK%U%!Y{#GV8tuEu3C44__208BOmCo7EET4c&I@};%_9Wc9BYGx2`?W0lous zq6vFt_K#2g!Xjks;i+5dC>?vm?~~DKf^>q&xGWOfBlw z^{0h-n&f}JC=-;&QxYSBltvL{5XN!J!t5A}uKMy4WLUygI<;3~Wvz4S_gtVUbXh`W zAJD<;&#vkCp%O&ENs^m+Bs+I$i6QJtD5*LSHTj-B>y^nHoj`lFF$7s!9RP zcG#(=kH*N@S_}&^j4yh-%u)>k3M)orwtUZo6VDLve#*HKF35)y3rg++o)g1d8eM}r zBx#489C!)f}nf?JEH#YWgz6Qr@1Vo}uXDZ8;$xT(u= zHUFw;>Fkx(?x7p>vQ(`N4$6aF)Kju38MeX%dz73Gu~eR3TbM`k@0o{C<&)$|wXc;8imuDC}6#a4~-YfO8xQ z^i+{;3H-|W35W6p3<}GOiSi1$P{4oU^n5aU!`A@F>lfVnr}whjC2pjJvpGFO7%6=@ zZF8TGkmgPfP&0&y-AKeNT62DC+7j{hH;QL8(cSiAXm}@%yB#OzIM_-&}d!06^PJ=2r}Xhee0}Nfp-S(;ea9 z55X^aLpY~8C0eU2@|(oZE%naK+X*RIa7k_{%8@*RdH2pF3M$4>AydV#w!1$P8YoP~p*oJ7#9 zQO|+u>!yCl)hzd*PhZ*FJ9k-{Su${#ev6aygEvl&8@<%9hy$%aY&nD{dr_yR`NLhq zt8j6#quS2VzJDt2NqS0gq|QAq`x3e7KM);By)yluS&G!%#`>yzrd|3OYqi5HUy%>3 zaTM%v$R#qSESV)qb;TsB-@jqJs#RoCzVKw8AHh+GW6u4YY=iI?T-KNrEcBlfk(w_f zCBRCu5ks1wsv#C8Q+NUI1SA@&+H_jjhmTym)l=ZRD@`gHV|ulO&bP1j<7*mB5zv_* zg{4%(!Q_J?3XyP<>Uc0?TksU+#1I%X_6Y@!K>`+`R~24=UDWw7MlFXR{Dopb(&V>; zC7c`_Td=iJ*cDueRlvsBZ71mDBeQX1MofXyJ-QHTr)b}LZ+1D%J~5@YE>a%SUg&38 zoOkBqc27QJ%oe3ZP#tO@Me=Erx2_ckgfYoMi$oj3j1 z7m`HM*^uJLWE@gd$BPF-Tjvu!rww3iL=zRoV7X6$+Xj4mYnov%x&K~8SZ+|Q9?vfB zMN~A%P=QY!-CK6u(7R0n@YV+)3R2urVSRLpfa1ZTkVD&SGcW=kxq9Q}r`({f*5LG; z*6yC}?K_jR=)WTN-?X3ln4h?7dnKd4I>MAcz7sFu6lH^Nb^!zL|2)fjBY*2~Dra-O zh2tC#WVdD__JCvT-cTmh&z`FC^0ph4AK;$W)V|>}rJ3zFt^Thw`4DHH+ubiIBhsTI z-Rxyp&LH+4R)@GxG_EcGI(>x?vY~Mt7`_N6@kS@qqa3UPVN z30-odXph$8%IHg{pII4P&TG0(mUC&lxPu26eT>0x5y1&e;)$?FDIlm!v0dQavo{

    qnmJ6#U ztUG}k${{jCRWW5oUmi7(x3>#c+d64=~8GZfbeAs)voALf;#QB{b<$e7B z%fGl&BPfLmB$cy3Pmv%|#wuGOQ5K{K28Kqq<_`6hmTPT6Eab^4B$YQQJnDL$_jmU{ zodH=uby}jY`2$5=g;)ZP!H0mQ!acWH84{HKxIrOX5=xX0KDA#vJ`V3zBl9yl+_`Ki z(6gorX_w7bjmp<@fCwcy_J_cffTuV+3hZZT=|W9!XfYK(u#m3Utd(2#M7cd&JWu0C zMwR3o*73*>9Q-U=ym})`-B$e?tV+F4kwG4ZQ!L*N5xusbRuSU*7}9U%><88jZCygE zT1*V`=r@_$5n$3Dpaswr#Fd-PSxY>pnyEVdh;R@3$fZ=3(|8au8#8OB=Oy15T`1>= zy#w)es|wHcmAcxf>N<4xISmYKwkRH-^Ea!_25 zBONMOk%J$En^aXYz8AI>Qs!?Et_RP%%Nnorqo@u0DXA&x&;4XyQn7b+_vXby2Z=K? zGX(z^CXQu+Hv3G+ZYVA8-E)cTXl zDcu&Q^uQ<1@wzSmaPAW!H1QXWgsFmrKHnUTByr-~fju#q3TXxFbmP`P1@c(a59o)t z7{U3T>rr;tCVYC5j{+m>&nM3YeiMHGu{QuNS&5ib@u`w4Idn6xFObKROxq{!wVa)O zSk+Dx3BoNl0!gjBAa&SV*y`c}Z%d?X<{ke`j!Oxu_h-U$U-fC@$_R$4d4Jn5M0a4M z=VelPoT{7MKbNo`e4!g!uNTn?kt2s$S>Sv-$!33GhG=3A1@}zJ0jOi*dgJP#45m;< zJWMdYChY&ry8~9>GbuAib{#_56d4D06P5xRA;FL2wM3Dom-w7%KIb}U1_yNIDtm1g zRL4sU#{?>504y#2^2-!vf8Q3~6+4dG>N6+r$~8_5MJe62?PNS&@ei+wMRvWEmaK=P zsj1wv_DOIAipnkG$Ob6MmGK#_e+7NPa`@4Z=W^DN@<*^s@IdM&W8)Z!G$lgwdf9VR0ED_Jx0NTnbOM!mCo_9Hfio^ z$;OMK(vmgh5G z>kysq>z$V=+RtwKZWmQ%3RuOopJV;ZuQlA{)Lopyx?gf`-5Z8w?LMVi=7`!V3Uw*9 zMR4N8PxWoq1RL*7WeC4GK$#=aU>IsrC9=if2)I9Vg&O%`4ExEt zg*WaoKdQv9YJ^jDiXD%~?{15)g7VYShHc%so-U*LbS8g8L6hKrzOi+N@aN&GY(=rs zsMu?#fkwC%{o+RH)C_u2w?APf7pU%KMeE?0LiU2_^&vqYJ}BW%C@i$t8+A;c==C+D zcO1F&X8EqUEi*0avM>)-he!^V(4U5V zzim+L%B4{xf5dvJ-XxC}b~~S^c+?QA{0Kisk7u@K<#=515kIgobvlke+HlDnv@!Fv z`h1UQy3zves^~^s0(z)@P+41{N3x@s*8u;-FczTDT9M?KBQg_Qyd`8?KQWj6a+;}U zm##EHfI(5T&VedRX`Y@#&er@Ij!SV5T*%&y1hCK=_t8#;weeu=0{ktqh2jMhY|0ch zC1TGZabdjCiR`Dpv;yIHNuH)&HW%^!MJ7#HfB&8ez+N}jY|@CxmTbX)ej^nM(m1!L z5q4-9?4lycN5_b#Z=a8y*0>>9W{0@3^s&_^%IS(X-W4!~+C{VFSLGi!`YbI~Wch*? z{9dohOX2`$x(fGf_-#bo>+!cTQ5zBPSzb!}!eldNmy4HZ$pN{G1^RZ>>bim=L%r)hp{;QCKF>E#v40K6hb9bpNun)6v#6b9SgHo?F(aISpF z6!>;`s}3`Ea?P$EVd$0lJsUV(RAvQ$5LM*{6wnJ==k088ey;!Hi(|;)Cpsz98lCa(85%psgKIOd+8;SywTTmSsj3sTqXx>t#+EJ}a>UE`yHj4ImMXZ9 zHkeT*C?4g@E(*yHd#CINSW9-1$wG=}`0lW9q}RW$2nd1fnOGW)@${5&kbf5*NcK^v8i0%;79%dlD$X{j{XiCX#0WZR<6PqBcVpSwzdr`mz8SO41zR&8hz7E z=R;5{^gjO6T3Z*xedv$FoAheGSNd&e_A-XcOjNpkJI&p&-Gm(@>2yx0^4+W`RbyE9 zB?V&a^xQi|(0mdCZqF_kYyD(%aK&;oAZ>fSPr4e=~k=|vU~kpb}5`c6?FW%O05*4VnV(1DE6S-QIa&UdGz1v z2f_-%mygHxz`GK`v|-C8%_rmKOn}?`k@bhVpvsBYH^{XrZC2rg7 zsZ{FF-GM_-syo60K1iH4=~0Do*w}T+cImlKpwwIYE~$Uh4`go!?X-l|cKJ*o&Iu*l zmO%x_MFQXVn%?;b-o)BGFLbfZMfHTP0+Jy?iI*y&ZbA28;yV^jyJnF+GhesUC*&|@ zC8`J2q`L(Tw846TH9IrN=(CX z_Wu7U+YZ{G)gxegy@ zeUMZtr}?QyeahA&A1UDdt>N4rhG%lx5iM0MRAg0^!R>dnCK@?1;Q&gQQR*{s^wA0A zX|Q&_*t1EeF>V=Hx;Cs=3|O8jjvWA+o_DR?oC)f*8~jc`kKtJ20Q9I?mDx_tbjR+D zk$Bmmdp|v<<+sLDe#05WGTPLR_2G4vn8keI^#%Q8+T70HJ6TlY*0Y5<`(Cs07<;)d zGG5)_fH>7P^y53JrSf2$S$3^ z*=e?>aQ(6U2T7B#l0x!LN*Kl%JquuPCZp&||Jdh_hk`1sNequzqQOE%2WiN8`1qYl zF}rt4bjC!fiZdZMH(PIj7hp} z`N{=50M}zXF|+qqabW_?wU|Xn7aSDn13wgOy#}%F1MMh-Znu%xJ;3AGk`aT{8lteW z)2NSTVJ=1@uqR}nDPV?@w_<%3cVq0g%=TYz^mSBSKj*RVC zXjYzM+Uk_Sr=-uFWOM$!?9Z>YPx+tMfwSvJ`l%WB<(cxQhv9e57QeLTo1-&*xu1>K z%9Fo|3=1trUuZKyM3rB5kOrZtGO#gjlpLZcKP%dhf`{#Vj%3RQU!vLoOE4ojHVrd) z(rMaiHs*fx3vkXbM;fMjKKus5m&?(B1qgOv>qM4=5N6wT|0!wrSU9c3i_(@RpUlW7 zWmse67}S)ibDD)GXA@=o)fm^^j?(YD8Ryj>-~x6ITZp#~QE*7gF!kY11`^Y> zP9YK@;&eJJgFXWu`4N|Gkgy5#N&$6cOrt9y6*XE68uRfa?0p9*G1A3ao(Tb@@>kjk zH|AKGtX8EIR@D_HdQz!cNz06+hti$%^-M3{+P`EuRZ5ELE6h)BWMLY=^eprUWfdo7 zE|?)gN4OX>G%n;I;1+Ye!w{>}Et;Um&;e=%S3DhJ2LoxHcCo+j8^huG32OuXLg$-Y zMFGp7$ltYqR)1fep9j?uZNccljr!BAYCf&?*3#Dr9h0J#;J385m90XG3IA~xe#iHN zJ8A8}HzvB{>hiG&LqSCBsOuhR(%v0S$zE}g?9DGQJG_4SMNQdM!&Y#}?>TYlLiv^P z71=&LrLqXt^Kq+oFvC7;C+EC}5Azl6ug?=)tGx;AIuol&5HJ%6-LcWBr!c%|^QTsuzZRTh-fUrD$%wrX!SXh_JF2C;aBk~yyeGTxd{--k z%ePS(Q8yZIDruIjX9Yf6*r}Zpve(%FxpcXitAUh;RPC|65eH$yPo>xB7^$jA9*VXj z@;+A9^Ah1yZ_TDjv)M@^x$iO#IHLN{*&>kHOA zpKaisOpTXDLnCT48fLNIQ)tHz)~EiBIJR`Tx*3sm&LfLVw0)+1bvFN%$r|9??%`$R zFoqG*R&TTFu!sWc_$Ro)mfS_$AOk8rf-Pi1PjpabhwA4c=F|7=PIjp#@FNcS`Oew_ zOJ4gTd-pHYI92Tuy;BS|L_MAlk$kDVGID0LN!$yR1iyIJjcZZ5kMPeVR8J8FeX+Iv zH>eCdaJNTnO{QkDIc>6%ygd3=R@4AGK-{hpi%Rk|a-OW`pvaTNXaHm)+GO`Iq}=OL;y~_d3j@vopFI?IF&hC+l?N;dN|*A3Xu0DHPl?hu zy$!fieyFQzf2oGzUgW8=!kdmi%AcgmnNf1p{*(Fld{YAx@NjUmD-ig4P_x;BUX?7b zw6i7vIRfcdmGpQ>PrJG@n%ZurxY9*(Q0#Bo2;IGZGW}>Fj>zusAnV-scr9hg*}Jlw zuGgZ0_-1)c$#w_Z{5Ig;XvU? z5*m`|HX}EM>oX-q@?bwo6Y;St12#P+ppu|*NNf0`W>`4j=^kI;;ctb*etzVffg4x@ zS6R*)qY)Yx%ld+T~SpiQllo= zB)d7QT)ZSN+WZ)A)RkTPWoUPEB1IAhBHp_(D8aQg%g{vBN_?uZz$! z0+9jqckgaqjRfOz_iPz1SXf0bq8f!-LXvH^qdSeeA{lrGDFP*`0){+DO`Tu359Ys2 z!u~~Z%PnxIHqQ*LIs#gqU!J22?dMNUK!qv3pMMmFwhQ*)(tJtHR*nY+2}WRNyW?X? z+ZPXTA;`3Og>^Zp6OsN=A%IENLXwgri1ABE-XfCMi5TS(p!M)RzZVUj4+Md5`K0#8 zL$0-hW?F8X0P(wo%x?wOk{j*?c=KPGTwqfXPQw=ou8{M}Qfl=`vt*F{fvnpj%i#LV zGX`D^qnz+RneVVC5GXAp36w9V|p?{drdi#65py4bQ)y*)FKG$F{;q`Q1?y>rPn z5R#;M*%cn&Ctm()`ZgurGwsC^<+m&Fc3B0_c+M;b9(dn!2q8WTxLo(FjUcYrvQ*eS z9ja@|iP#&N;C}*KC6}KGu~m8vt#{pe;v3_5p72gga;GNOiTVk(iMKo0A-kF#+<8mu zHUHrPQ8IG;(X25Cj5&7@?ZunA1owNPOkEg#A4c;Q>$rJf@ z`1Q;kKitgT^%YO0Cj!u)fKH1AZ8K1RLbCgQ7^iYJ)$02Mvwovz19`H3^WF~$1D#GB zGtSb}ZVFKd@5P(m5&xhChW6jb^>ieNtXfq&RJ=9VEufBn{h(2TKA8o*R3DfHMOcDl zc=Dt6WSr03u`b|R8lsMWfw9!d3LpSYeK>+ubz;@gu{uLSgPGbmG zJ`47Mm@qcG%9#1Zb&!L4*XP)(y83ZU`jztzRaWi(nbEkaADSbAMe)EE&eZ>iTO^Nq z*Y_;teMLb9FoNApvlK%3&0tm0Cn zizouVarWERdPsB|QC5dMOID+}4qKkpU>LIatL3)@T43gaf9^_ei`4z6=T$2-;UV5d z>d;`dDz)o_K3bx;ST{3vBGIQrkZFnP(r-fM4&E9pSmu1do3hEAJPB8))NDdSrnkdX zOryD4N(XSG0h_BJ6y*`pUB14#q^i+(1hBd`X}neGly@0sA$=@sSa*Z}UZoko|)J;5)Bdy{K{48~0V!x76?#d@|q z3_`1s-G}TkTyTN@+-2SFv)DbFOPPsu#-Cy-2k`SLYKS$kaCx&HSF@f`XSrGqJdF4q zN9_q$4K+4WT8Szp0dRVi7Vb-b_XUX>>i(wLtB=Kv

    Zle9^Ema2v8F)IzK(-- zWB+U4ps06E9ITXgX7D9YQE{=Z(@d8;uDgUcHErXJ)!<-8$9~3(@0!1*HvN^a&b{sn zCj)K6OU2nh9Cbc%qqR;Za;CJf1B~-oE=#JIgEjRHBBaNQlUbxCqAqc7?>CEsG{Z_& zoCUF+{^NDzl&LhsduNcT2#}Vlq%ZH-uQ^%s2-91N?nP8&tCNrxS5`ptId~?S+@#|d z#kRu@{bRUsv!$S3a#M_JeRuLBdQd(aFl&HAjF3xPH(C&bu!Hr4a9EGP=fT5RlV;tn zp25%COxEvj0?$QV+B7Yg0>#yY*?0YOmfY%WuRzj-k1k+^D$&@lufcFWyF@6|O?1c! z98q0LruAi?;1>P?wA(>d$_@zXn5AfBM$WLkC4spDQsi`{i+WJZo#-nW{zN>HZgfV; z?7A9{S={dza~F~F}{b6q1)qR+dPJ=H5m zrkg+jmt}-Naln6ub%da8&JtyhbA&*#VGPxK&=SDEI%g@8Rf5xf#W5xl~_E^U-EzC>0(_@4%=Mz2*2U-h%5dgXNcH>iGIa~&SveDZsLmZH=l z`P-O7;b+T0FqaZ)Nu<+>t}rsdy-ccywG6k0jA-3MlPP%&$?*=c9h6?w=5s4bO{o3a zD`Z*1ItL{i&v)!TVi#c3uKdb3H)`wa6#D>jIo!Vl0zd}JL5H2ToDL(fTKjx57XTBN z)|Ll8zaN}Eoq~sTJK@oXUO3?}^#PTsnI>N3q2t&6Q#m~DT1KrXW>b@?RIZK1mneY2 zVVwrw~p3qn|z?;Ve+$uJoY_$-%ijcxYdh~DA?y#|ql=+qw&k*GSubd%3h zL-rXr$UUajudXfO{FQGJl{xp=Ms>YA(Ax68PNQ~@Y?2Y?ELKXw!re||J6x}mtUFq3 zYLU8bguccCmp`zAx>Fg~DBf`y^46AVUs!!vMW7^TrV6B69&GYWo~J*4 zsFqcBY)GW(DdAHcV{q~yNmp#7r?)1bM2_*a?qzl!S%=kbyo_%EQIWJBfx!_#Np}Jc zUWCvf_YlwsB`tMZK?3c*UHpr1w`oen({+&^_fuBDMrRdHP{oAtSN>I9Bi zrR_P+KO(!Fz2%^K`UIY7H&+_?OuDb>LkpvOXD|eg8v`J(CRSJHF;==5#m!j4M+Q9D z=%RCX`MC!uuxO}Pr?BSSR5!=eBOk?^V;3YZi%-gAM6vi$*7%rd#a@pEo)hNI{5~BC z=JKiNH*)#mnA(Sm6NtX-gN`8UUh1=GTjtMR*iMr#aNPw{3zq=L{A7L!=iiCT6bxpo zeATkkKNmkAEHRB{)VNIB`J1h-pabShF063Mwdn%5L2&iwW?J?HKIgNSLxXsB6?S$i za*m<>UF&AKXEI^C_uOCQuwhHwJ2D(Z_gwxU{%HhpEsnhHz9ov40LIT}(((a7R@>J}}ar|k`xgWDE1$VfdGe>KAT ztoY^1=4e?CEr9`QrWLnR_B)?mfieUXC^$*epa}milwd$R4calS17eBDw$k;wZZkwg z4<+}^9sEh-K0XX%{De7td!?9at3w*H;AN7IpMFe7?#NuZOd2h=zL3Tfu%;jNq(bk1 zb^~eb8dr%3X@;?VMfA>oCW)Un%hg7r50{PHX9!+59uAZYvtoYDAHpWk!q44f+_@QR zy|d*oBnQn%rifk8>$@tD5TbX6%rK-a2TWakB&Upt!$-quced^@XZ4abI2#FXCAAd6 ziydkw9kbJ>uw3k&B5X|1B#)srJtp|Y=NN3DAi%I6Nm&-i{k`#ZgjiNaNg4Qiy0fN< z>G(6C82IC>e8%}!m#6qMyZc4jn6$Li*ADhD>+)Eb8RHVq>Lp#d7(BN2JkcxO8`7uO z$iR~ZUy$N>HF>B|TQkn$T*k02Rp+o~nzR&ByU~4Nw~I1fHuSUBPtH{B#$(%es{LsM zmbq)agk|VkHX}`M4=ueQFZJu7bCw*XNA2Iqrd=6goiVr)b-jU@S9lG)2L4AR3N5b zw8M(LqHT?-!*ROh9?aI?&l&e{IZ=5jk6k) zk!O7Ds>JRyCdJ<*&M&8sQ!95IG-J~Da3~GaLh7=;Tou~&K zwG@&{rykfvX7lvkRI`?f`F4~*cYzf>CNc*oq6fVOKNRxhODsG5i{vpSJ3+};_Jnoe zD9$cmA0D~?i8Q;R9iN3!1F;64T+RU@Jxd4fWeBPic*cCRi~M(wCD6e{zhB>kF-K0{eRM$%Vuw_0$?Xv zDx_bPd*nk?UcHUBx#%V;WV6jjh`I&K;_QG5r#hvo^}-$P<^5l6H7k0<+z;8umX_8@ zkFuw&O4hK$m-3m^?y!(qmE!2t{!TVW9fZKA zcw-Rf!pRBX+_WCWyQMfF<(o_!*6lR!KJXOt$-?h6ufSo%(aY#o$Z5L4T0ueD|%)mjqso;AK?HZYY z$h%4M&`vB}sQXYpOt|gSaQR>d<{{rG=x!~)_puSy_AKx&Tpig!l^8}0Aj$`tA{B^> z&#Yhn`_rNKF3x{-v%AS{It|OVDMqk1TN~dR&WkdMdT4smx1wa-OcS@Iz;b)fm;#F7v2z%tVEa_OA#73cn=Yp?D3^AHK;kqSPuF5&4YKj z`OQ4c^bD}oyGPy=s6YKSf`9riajwbg6^^Y@v<}|m7D}OYd_z_VyeO=0pmlgXZ1V6W zv-4Xht!}>09>Z2&j#Rko84$oqaUxp}&iM(SRwPMfA)ItRKEoIRKNNjGY)?z(0&Cy{8Ybw(SCoe?tfc>Fsa+{`MpiU zodhn%Z9^|Ismu`NT!h+jbLzO~uXo2ZdMWyEalQyj)b7H1hMY85zlD$Wzl>-Xd(vm} zWE$+q3~>>ytitOceU!$863R5l7tMJamM<%z3F=bS@{>4~+KAzihlu66QDXC^4 ztFrKbY*C|gyyLo3H8FB%$dmyc>@&Q$Q;e|$Eo}PJ@7a@=o-=bULurwLbG#Jv*&|J- znw8vCc^s{mIAsx?bv0{{OnLmVMNhkXpsRV^>(lDh>LpNon;4v)$Cyi>t6wG9Y#Ji{ z6kv(-^WZZsdRh6zMcJbPxrdDtI32_EB2+iip7X;BnT~xXxFmmLJ>->F>oqHeFC*@7 zl`fSQMUpCy;-Rm`XX{V~tRF6RBOKmKfW$Azdk8)ninJhtDyrCJ7Z8o7vzVR7YV)XD9qsuu@O;gJv2Za<4&2IxDOX z$tJ6!F<{F$=OF0PAskX<5ZlBZ)*}2T6QB>+O!Azs5|6d|#|DZ~rM%5rU0*V_JY4*z z4d_bC(vEf!yaSDU2@<(g(&Ps0?=|OhvN=-Lu?@28t*~B?Dm-n8u2r|s5|x(EN*-?i zzZ%^7V^iVWBWO%dr2XgT_p8yj(%$b&l(e|+{x-6zRX%cACT5;ZHJKmjfmWVfq zg>HhT?{g;f_>7--RnyAUu=sTDb$+AHRfy6sOD|MeHVMmC6VFT7hC<6_O`CJs{xXY@1+-jQtmlI zNjWC<`aP%XD&qA%X6eqZ+YixlXPU;e_a>jq@NB;9gnr3*RWEbeq@C>KiYa{qG*S&} zLOHpKI zRCPmK+f5{^Z2z*}xH;-CVU7KwXP?QBj|GdCn-?+ik43l5gRmS+4?fkOgjYU>;SK{CfTLzF@^W+TG zB>V-KMrt082DEIwjMfVLppO~wYH#kYqrYCo!IKnhNQ_T*5%}6J{!WM3f(62VF=PvYx3YO@ zp64?qv5;^U&Z@Lvlml$$aJ$P)(fRCYCbvNjH*YjTwxb^`fZv1v{+3aoi&^X}+ORGy zFtv~6cBgl$Un^d{qxZA*hCf!mc( zX?uaCwQQq9RDop_#MQ|c-%IDKHVCw2lt=SpJpk6ioB-(XIw5@EqZLwFb7~TSA`6T7 zrTShzSl?i?k(Oxj0cCq5AE5%>gVtOB!UdwMhSV}M+u?D|lzj9)JB|0@`Q0|peq$Y4 z@vRaaL6JEKOdpziD&2}N}uy4S~wPQ z?Z@ePW}M}1FZUoN|ZFvd0w6p#Gma4!g*O9T93!X^k=qE4{&u5AhWtM>$cceqc$)g27T&;MdLKRcc zB8f9lM{o`jY@*l`j!vLBUdpjBKm3~hAL zXgPfWp5Ck9Nau?YspmpUfs|}3KT>(u(`mTG4&%|lu-cC{s*IS?D2bIUU0m|u%HEm&L;cAX$MxAfV3rr+jq`gRl<>zet(U?)Q$HMU%Jd+*_XdRU z^XPPm7|*4afNyo-KLeo@rRWai6ymd|jyITO>U>m@HOI7{AGR}ttm)*o_@-l(@`bo9 zpgH5(X~ao=NEZ#^C6`meK>h)uj@q~}t5N^Ah>v$`rSWIwDgk$6p3< z(++cu8$#Yl%F{K1pD!)%cW%P7y3Z!18Y{QL_!*56(dc)PuhpHXC-PS8cDn5<+LJt9 zHeYwFaWOW->_D2SKVl2ULs<~|nz`5=>KSW2clEWfQ;uv8yloMDFwDZ1H?CNgux1eW zgF6CA=*ya)kgOha$zsdm?0zObz~SGB5mDx^HNnWXF_rGU)j_IJ=C|a+1)WIw7a?sUgGDi5kz0|?( zGDiO0V5`FGf0ZIm&Ch5NmEy_U;N>2S^0ToD?CF%^Au%lT(?x?^I_tSW;|Rb~mcmD{ zQR2oxPM(aMAu{n`Q;vd0u&un|lxBhGX79AdfKILl4&khoOoLqxjhrJO1c6WB;M^Bk zDfmr)GoIkiSTC^67FkE*9sd&@-9N?72;HAM=v-e(Ncln!F5WXs$?gYp2{?!m|84y_ zA^Hnr$g0f_K8Q;RlxikBMez=akE>+vC#b$?`EPfg1zu!*-Ta(42>Y&u$Ba6i=V}k= z6>(Y|Emg~^LRpzVSIJxcfv1Wqf||}#;vguZE)GvA8emZz6cD+oZ~T-v%K`~r5%-h? zJ(C}l-cCCmm8JJDdB8COBUb}WqI%dy|6R-;haQDo_r9{=lAGQp)0`J%@iinH(}u|9 zFB=y`&<-=gKpRdU>aKEnIA+VlMOj;lr5m>t5zs{0eFTT8ej=^#k9w1fbT}(#2S4S>7MUBGZtNc2+JsXo@qx9#0F9UI-oz5!NWlsvr z;_ljvj8-kt(qC=V*~4>E^ec%FXeu%q?&|3bqZ*B|%G_ezQejOv$CYb*6sZqyaRINs zQgwY|?wE|}+7kp3v6LEw1VOUN({qCsZ0u!912dXh|~v^ z=3e}VDT)Gv#za<~H#6rW%GKb{u`&4hSb~JqU1Kxi|JJ2a!zej!qn%nZ<@4WJ_D zb0)8n`iswi>01{#zdiT?iCw^txddS+Qg23-Uerxo6#bDN7kx) zu@+9A6y%^{UIs$0Sm(w-sKGE9?hpED5)@(ef}}1}JdhIvl8CO6=UonX<%-UBcK{#F z+WidwGo39t3@qw(e|M06Mkjf?^Ol6bH}BoS(i}g%endruUzd-j_c1~RMMZ_5pP8<@ zI)>e^(7d0o*TPxHJ43;|ubiHDNi)*2vcL^UGgd{(v3+rN%o9AuS=p#HoVvwC3}y`y-c`0_egeYrduyyN{}GM{0$7Xsz|73*`=Sh{Xj%%~q$P)-n zI7&M)ArP+L59NLXd^{QLIScCvRG$Snt@mP9PSG=Zkqip4txiK|f^pO3W5immGty&^ zApWEuBv|-*N+)4;kZns(IZm*BG{!}?MWNC0;iqN*E!i7izw%-K2)q;d}EBY z%jdk66yOQRA4~X|u>ByO?T^k^T{F8}iH6Obg^gN9F`o)GLp=fSnmBSKI_M2%V=HtM zk{j7ltMCj~vdBV@lHzBO#C5st&PCw@W5ujQ9GELN5Fxl%wml0w2^wN43IO}ya4!0} zwW=q|TC>Z56*8(RVv*214xS8Q6T;bJG13aEYc?n-J0VTUlg3@jGD>wNcm=GRPceMj zdi=2-O~sC-P!HR1xwb-dLC2IF87JKeQ^7v1olpDQPwMZr?F!T%VxVnVY6Mr%lroG& zS)=+?@mzn)X6+_z*J@5Q*4Z{{HWQl6E1jh+mIO?PP5Z?4RKcw0}Sr9V3pk$SO|9KME&J%fFq zW&x@!RCO7bGMpj1nyWN$vmrGfx*)oc;-45!Kno+hZeFVabXaz3zWO2TfCltL0X%G( z%F-Zw55L~%HixV=*+k$F@JvD1`=ipzG1Wgpfd%k1=@iO&J^)Ao6_IYHmgHnB06h@p|_ z(T`!d>PLVLs3W0lN(c)ASDjyz9LFc|)0t zj8wnljJR+6dl6x*pG~S2$stYR5q_{u!NO3fUw9vli;SDTc3Tirgu7w$dg?b6)9ft5 z;e|u4`n`WHnjG%kduj4(=an(rhSqrFsY_w`GESPs7O~Mz9D67LRv(P8Zmh**>a;2l zO>v0qLFeOQVJJ9qEg}Vf)0D6=Kb8M^ZGj0uvdQDVb{V!MD3ACd3JnnuFeMTGP>lS+ zXLzJKw07WesK8K4$tP%_-(;d3*RZ>q-iPLF|6GB9;28?LrQl)6?GVj?NKA6)%39qI z1~ikBGgtRq^pfix7YD&0mt=5qt>aCKi(#%yk@Bxo08fS6f?*PXc-U%m1ShAvCo`@7 z-Gp<$K)D`2T7&qOH3s2stY+7}phqVHE$s@4P$a*AN%-`7tOf9gIDvQcwPy*>!3(jZ|SxbY{5r_hS+pU?C`*xxFp?CQtG}U*lH%3ujKP zdq*9gP*tAo@_1Ls_^!P~PPw{yEgi<(8I~j_Zxd+w5w@B-ZasoppMH;yEFPO3@|jjd zZlEuC#3RNG6o!V%0n165?0F-b06Mkw!z!aS0*{^BLk=!J{C4OX-m4--MvYC%r+JA& zPm}zRY7yAzJQ9y{y$eA*^B4&aq}w_M6ql$?acklr>Q1TSMpz9+tU*C+wTjwVq^mxx zW8WZdX?Z!i+d!xK8;I4C8dKo?=uRvHfqgfch>0u_Hz%zFxY5Wud|dkJCO$9jUdhnl zfhKVo>i3LfDh4+;rx}){2lW8e8#U$2nStDsVT_;0q4mxqyK9u*g zD(#r_w0iq!sO{K7mIYe2E+wu{n#H~WT33#KiQqzqFD?bW2|`aF1=7=1)FmF=uG+1I zOZx`7bhSqQJjl6`unIZwU0L+}RI~P{Tp|JM92kvgpeO zKIyht+;k@*`9i7BMiiDjW6$NFymQ;%4bHA+$rfqedtyAIjeIxcEZJn&Q!c1@~DnV zJayZKt>~3{V;79yyvK_`^midL^KN^2ODTqz(;d56xF@5lZBh8w_;nv_;NrVgYBDOl z1XZK**G<7OR4+t|4ah#kx+9UIJUkMMSh>82+)@ZKJRD3~Na@MEqWG(r&wr~G$V6go4ZGR<|4<`;(a({{q|;* z{;~&e^!b#R1lU@ChcM3im=ksDS>WZuIHlHUy9yEpH_9Wyg%1*i9~2Vjz=y`FlPU7j8ZT%S>z$yK?;Tz}Cn39ddKfW5REpLC5yE~{V0rrUoNmDj=k)VpMlw`z^i1x-RpKO>`mr>EtG zCM;a8C_3W9l<#G?kLWUdX%K7P85f!wxUh{j23-l4Th|b8b?-oDahPsi@8UPAUg(gZ zXhCS=J-P+910_4pV4G`V$d2`JZeGV`p0zyS0`AFdbRS^0QmCoJprYAY#k#tct%#iG63`@x|h^$`dTC8~ZBK2b5 zV*7;;x@*!yYCJ(#^R^?aTy}yL`s2|Gy<^%O-PMsZ+E<`#W6+QoTrJ@k2c96O*Qtb; zJtB-S?#$RQMt)N~11cwDT=UO&NEgOm8e|zlww>BowIt}SwYFeUmn9KiKPp> zDO*_Q`@>I7dFfFCgXA_CAYn9Q(DLG&)mWKXGx6K=*`Ot5QjvnBH8p&9whj1C$0c4S zjrrzvR-q+1G}{SP%_8_~iBPdcUKs>ar}ZbXRr zS$UBcwTLN^I2S|%$W@yHOs83i&7#;gSz)`mIrsqiRmku0$pTjh1PmD5IeL2_F)gkb z|1^k3Vc+OUMND~XT!mB6SdF*2$OB5!nfpN1j%p%5nlEyRUuL6j8h@%Ef0=pX0lF+2 zp|?z%9|9^}|9;o_(qF4oD4Bgk`V<_3~*kkm|g=Mv3vALt}3il-no6Jdk!?IM5z(H8>Rx1YFcT2G-1 z7TO_m8#+n;nT;+%YlAUf#_VbD!e4`ba&a@GMoVftblRg1$ZO+GPR&-EANib{GmVFM ziWrG;dEA61qOqrG{5Y1uSNWa=t45M7$zcxbv$_u3eB3)B6kGXx8*QN+K(aa=%;m@h z)X+)5)}d*^fYfRl?k@ANG+BdGv>;|@!{DlLxz4|%f>5dTuyx%qUgM0bek z+}W`udt}+G5tfrvAKhnEUzT}L0=#(^GtDKN(% zD9_Z&s*!Eph_#}F&`maxE#_Me(RXMn^&aZKk>FGKJPr-T|$L?GmVj2p_7>Z?&{7r2fgIraa6J zros*ZPfCzqIe>^&Lds9TwKAYA{3n|Bmi6UCJ zDrP%%;R;am5|EXuSdoBMnQ(t;6bF8yLLOS!U0<_4F-qbEYA~}u2~lYhkzB(Q2~JQ& zxF%qM^{79+b6!SSWvdRZ8;ucs1cU)U(+vxrft5hZf~|bo5}xYOh$%x6xbkTOgeGpV zCk9+4bN0>gDpn9fYmq9S3^W*xbX_c%O#4m}***d|d3#OMA*?`=VAzsLfc1VI|>iHakJA7|ABE;b2V5p975S_JDx7*)LG zgXyiXVP2fnDCUqjW|W>3m~}_1y>I~w)6^JXLJZ)k7hOR07(~4hMG$X$=b9HQ!t}r+ zQ)a_kkeX&d2_b1@;yO$y!Hp>2({#&1fGRZBDmd4O+>eAH4lmU8LwcH*S}y=4iX0vI z7aNQN2*<_&XLbWRmYLp`7=XdAe2?ACX;2cuA)_+EN@5cE1QKReg%g2#9tb{^n-F-* z7dFa|vWP4QB_LDaZ)NlMVp`!s%3lBy!uQJKvOTVY`$6@7M*#cg2D+oHo{xq2gf9As z1N(?}%J;)b%o~Gz!$!Ofjlr?cH$4l@HG-}R^mr? zz_bSiL|j0#ZOownBF>>1s6gIggU_I6b9oQbvgi2WTb*_ID~Md{)zIs zMs03w%n&juOqjHj%VPDJyYMv=EW*>5gxo>LHt>?(r~={}MaEQC8ZRKyB~1{rKZ@di z```B3DV~{9)lwQ{Bq2AMAaHexGf>(@XikmW5hZ8wC8-dxF{_!;Y$@LrdB6z=dEe=2 zms52T9skCk!8a%a-y1d-ipxx;_$>fZK&`)Rb03+lqoen5YV1%A6)moBHShcbytkUs z1|+;uKagkOiU&<6bzNYk-F@kjG!iL}SLtXB!jOo-TGDSAGmKcI)H}qN&|sIKgi zAipp9aVq9-jUK(;Q_bIQq@_^|SW!FRica7m3uOWobXrLuHIE3-N=FJ@9*D}0qK~O8 z0=0($3vl*Zrvj>3B~t-iM1(N|UHIT?g`iU-GI>o9JtztTRi`ikL7{#IdMPIQEZD`i zKzk62oKoywh<1|{v^pSMDaV4=8mMJbj}ZH2y)#w=7@$XAfCo8dTP*P~IZej)FH69`4+rg(TvM2VW3 zOB5;l#}<0vKOqgx(o&S+QY$);I(B8&IjKL^N^2qu-h80UsQ+fGyK-}82wJi$S8hGB zlC0&*cR3(7yJ@V6fiJ-gG&k)EP}4t(MxI#ZTSYvh>!7;!R8rchq09hz$1()Ag|KI> zW|{NzU+L46KRDf=TU%zqdX5MC@qwHX^GAVTKA(BVM@d;}etaL`okQ-wWcoMyH;UU+ zlm;C?+TUJmDgR(D$f`4u3E<*x)?eB1MJd5C!OO?ORg#S)BvfFUttXTTVO=c#i0fxc zPt9C5iWxT!#c>+JtFg)Xw%5R#2;6<2`ee}b6 zs4>8aA8R*(Y=I1L@1`efP&m`HnKpH_s`Crz@GBfvvt~zL>qK4{|5gJlg5ow}+&Kyx zwU7vxph1@%SaGe>HoY4?tj2%abron}IGYZFbg8a?oW-eezEp=BX8M&M+2}78&Xy~7 zZ+D&K2{FdOh~Q%e@zA_IHNo@wE6rOAU8EG#Si6w&T0G}7Sn4cz-2OAP{w(Mm#tUJ% zCsCw1D|Qhmo!mHWu@kSDB{W+-8<-CvC&i{;qYxZr(cqwyq`bCP55tzh$6c_$UvZ^8xn)nPia^DTvKPp9*OMdBlI=ySdQ`P#j#mRtC47vXC+^`A_&SG z3X>LTDjq!-zV^J{)X|oMGM*qmm2Gf4{TrSv2vT#2!FC)yYEC7_Tj`<5t74Mq%}z!0 zVTP?)ngJehGcPCY#uSBh=2Ze}h*aJ{US$Byc8qL&yu<~KR#G<)jZ1G+$%|s;R$N}u z8M`u2BrZ=X7vV{(vuqbWAyh;vJ~Jle>xZYjRDDZE_5%?v7IW+I;L5Haa{~KevLZe? zX$RTBNyW@R35*J!o5P$t^+I~2s&I)im}-#GSFYR^P{(!UgnDK^*J3~1=nxW0F?pU@ zXbXEdklFAIA{W14;-alg4uk)>#y(lLp2-#ox;|Q%hJ@r zO+79bO-yi8N3;jDavQ4+IUtyL4Hr3jQ;A*lTO6D;oQ=-~)Ou`Qw@P?|6AcX3lm@`4uz>(80V^R&fxmF8u1%U!0)40tLPcv(TheL}4 z7mTG+ry^Lj9o~zIy(I71oxUe5GIoxj*{n_ATGepz)diDvZkNd1u;eDfB1#}%1kJw9 zSQKSiQJ&caqO#rNmNTwsxn_q76r@jm0OXqn3;uAA@I&=XA}Ej{zp1 z(G=60^)H8ZsG@7b`K)w4Nx+}0k1Bi)hj-wXdKL=r4ZtD=W^>iP+2k=%MBi)THS2uI zN`SdYO1d39=GdWTEln0BpoJ>J_-&ExG|A) z;zjlEUpkg&*jZ3*Xdwo~if&kiU6rj{zfKr14RmynD`<~Mp0b1v+PI9CQNtBwrVN#S^M15qOrwU<_`m&!TFw5EV}vF{)%4qjJdeJi79ki|`4o9n^M87&#@Is@$jw95a=~E4LhZ9W+0WJoo;AmGJv5Y4b zIbnl5j(i9=wt)TS2K32!??A92fvxG@_O1Xr`qxWN6%1=n*bC zN)w11g;4nB?q_1XcBxQauSNH4nFZ+4#on3AJYYdF`cViTdmk8EkE2~T*s>!ekc*T{ zaM|wWCCda&=TMgXmS#sj)|d0*Ei2a5<|TRVc`b@Rf>kVfO3gCss_8J9FP=##&e;I< z1jSCZjB_ko&|FqL%k#OZ(pSlJxhTXu6UohrXUvE295K_7oa9ACjuawXMzkfB>^yh+ z5kf*kUtSm%!vkxuRO@)je{zliGWuVV7sCo^m*4$WHTK%@I31dyl~f3|UXiQI*MG%M zrMg*Z?FN5e?kq-+G~56TH1#oFKp7~kLX1HsvML#MspOX6VDKs-Xy%8r5AV<4fBllY zQry#|c)&18s^{Y(x`;|i6I4XEtOTjTl&xi}6;C)Tk|v~t6%{X;HI%1F^d0TXiozhk zSg;Xy0Q8`!0ZpLXVYMMS%SDN(c@49I5eYZS(0PSXGKt7@tj${qa>LjbEt0sv4P-%H zXBFROu!GNEJab(Ko-;{yqAp1$inY?mk)A2uQP!Z_BN^)?oy-)aa5VWubw~audA@N( zpjd$y&we)X5~sU1cH;ET`Z=|J-dI29*3Z)Zy*&NP>2Qce;9jj+^$H}ub3NsVJN`Fu zN5raP1r})5<=R27Y|m{+jvU_I&wfp=c+NyUpjqrK$vX|~j6P*S?kI0^asU; zCH%~FU9zW;74h`nVvx!dpNCrgUEeLrRhse8RjY~|obi;NN+1Y!3XJkj;1*AEV7{kAN zniJ7H%7~E?+6ApP28i<28Aw^iP2ANh(Z21or;Vgx5r9Mwjk2QRiAUullRz0Uk4c~mjqZ0x*(ZT=5-2Buaw0pEK>7L8oEU*p8x;=_ zDU((#BP*VoZUv?w4+#+)2& zu04FAu^h)|vCcPacNySj0ZukFudq-Y9D5v}3-Y*6#NQ6!ZIRSA%+o$+em0huMR2Of zb&KOr%{s`{Q`hJIXSG4~K(`ALMAP)QSXjovr z>iGYrJg>@w+|cWWEPX5tEl>=t^PUdlbhXQ+uG3zygh@$D zu&}T~*5zK<$G4`G)=eSqB*t?(n$^8Rv=yGdCi}(&L1^Ht^A#x&39UPB*a_FD3sJh3 z6nxK`nDr`vL0D()Ed7HhSQV316T&qx4|+>^1_GK+`|;%V-x;8cb_h^Q1%jGge(?ak z$VQQZy^y`@ULeOco~#~`ah{flFhBQw%*6UDjCKzX{H<8)(D;5xwI@EE5LVRraO-n& zG`!w(HLg`Ye5dE+pVAR_7uqMWe#>rI%qFS1X6oHfKh=~i>tQZ52WiXkr#o5>FTxxn zrVm-5oO?h&x*Cy{!T=!Iz=C+rNC|lcnCPKF{Jd42y6bu@t#}wES_3&fZXyp2Dg;7& z$%;Co-eEAjARoSZ^~D!o{K=eLC?^4bGXb2sYV9q!4g6W;>bqG3u{eL8s^(s7zWifu zkS3c(>w@c)f?UpsbQxIk8QIK8IwN;8;*j+Ndxk=aDdra)%Vclh-WGI|QYt~}Z6>Ho z5Vd1>D`C>DO<=ZSCVSVpFUC+PUAw4gw&EJIj?4MT6luks zNZ}2!`lZRzl~sswkP;0n&0*puWNO97N`J6kH*gDiCw|&w%li`!w5kdcJH%Tn$;eVg z6q!?!z;7#vy7VKIJB6MWsZ(PNU>ZEgJqE`jO5$-WS2z+kquhtl*HfqAZO~NOO2E>t z2PJ0__9kAC4%#m^JG0QBJ0)$KOiNFbV^K@h;9u_wn)Aef`JN%VuOB1WF7~+SU05i9 znAerkAsgfn2)*{wZOJMF42JWR9z4__q+;*VS_|A3F_Zv)$BL@lLBg|06%Sf;-uC)> zV?DhwUeNvQMPPMp)|CNAu*V!|laQ@Q7XxR^HgVkn0-^t<~^u3{*)|3avFD zaDX5uG}vU+*(UL(<`|!$H|dbXnrUMYygF=O5e?w|fP?R`AbUr*-y{wsr(4H2NK4%v zjs*k`f&s?^0Y`%Xf0h8i2ZR51HHEjx>+qLL)|D)1F6siT0*T1CwYR#Bg?f#}uSe?} z9r5sKP7AtbO{oJpZwd6%kbdC3JciXzXYzABlOK#Q);pEFc)+PNv>BWZO%GfWk@Kq3 zWejM`5`57#2!{eG#!6z=cL+ctXr<9RGM?}%+u8mN(17o_w?;mI;W~AQUp><$SF3xgm+2hAveAw)blLXSPC zk|w-ZA7;7KANS1z9h1FZpB-fg(A`BC0#KW=k$8*((nsXcusZcN5xEbK0vuHEHd&U8 zLiOuAip4!TYdB~Obs4-S6RCbC6X|3k{bfv~dzbZ*h@KTA_xquIA*-v4x7mAem=#us zA;fIn_j?fw7ak-u5VH22QV(KG z!QWf28)&*cC_f9M&VJvu`c`mLr}bC%)$YTwpVFO!?DHFQBpw%YY@CuRDI5>mPsN>M6fy;|TIBsJV zU=-?_qgbPEmTs0Dzw}n(iPGqClBL+vH5|R@x%NLKDZ_f=Oai1-u8()18rW91VCu7& zm`sSLXqT@*KtmK%?gB*^X({R=)p+bpwZV720l&9u2`+#acpGhu{YzF+bswHDoHm1sAM$#w#d9u>^-1;IH2$$0`2jY4pxO<-IcszQ8oxdPF6IN z0PR&54dpsrkk^*g16~PFGZfV<>*a&)wl0(chqk@i&5?C##`x&32d=q2)uSVFf^Qy6 zQVLO>K6rcewBOus%jl^?c|XN5wxKof&bhZfo;CO@+t6DkiR3xYXc>ULt^~QnS&?O3 z5mwytQWP3Gvg9BpSWdA2cJlq%hacacy?f(>dDJC+&CH>i$KDLAuG) z0If6gfe^5|s%aKj19qQewdA)Cz5ik8%7!aC-}qGEmOg-Nk(olO0sLfJ7rGFsJhs18 zl&eL~-)qPz&FmQ|wjxDw`>JUibepQ8$p%F)-7O&CJCsMti%M5^QuMyj-~cu8&o^v0 zLugctw*L?7kiz?}U2X*5KbxQaW9Za`L?rArO1OKY(*dkv4@^%_X9^g=%>{XrZ#`wP zvBVX!KbmEoMx^HWmg1pZLZKSo>YBY5muhS3jLpaekc;*=a3|l3H}@>5E9VspUN4WK ztFv6+uw7I44(10q2`12{J+HbVkb7R0(Cg_~w;Q&@WQa-|*0@AeUAGgWIXor<_RT$) zmHZzjXd_?p0tct7k%_ifXQR40IAAts84l~vMK%MBTInVO!YUa+RsY9QNA`+BZunNG zOZq2Z!g$RHFoGA?2j=$XUp39T93Fq=Y*&Ab^_=-F%@k9sgwo~GgqG+$()h?y=tvqx zeVi>Ov}j0AdlDpykXu?-JgGBU5_M14qIB9t!&y+xF}-AoDAG7J@T&h+znH_8m0YAI zVBCH3yBFm|q?aDmX+ir73v#jlkAo!N~ZWftQ5LX!5Tx<^))kwfU0CvSF?FWPT({5IP;vP zSeoV3V7$wnteCTIhQAJiUeKU-+6x)pH6R%H=61y}?7w#uG%NMe%=4awh)aDF(UjFs zCkT;DbFj!i)%l~OWNWx%czX`KVG*@NhQz2!Y(*y4yUVRo&R&T^RwYn?5pvU3q{p&p ziAZP)Q!22nL*=ND$kE!w-qIz@F2U;~$~e+uW^kv7e1U{N*Gead>OOux;w-sX40JH^@mn0D-7V<|BU2}?4kyGx(VTiI9 zs_`ooj!79R7v43c^*+Q(q8XRqC-1I>R|~W&zM)^h{tYbS+56Z2B7zasySrr?`kQ5S zi4%z$x;_J@)VJnUGnB?=rkqDZk#z`W7E#m9$hN8pZ1vZ}GrDOddt9}SWFNUu>d}(F z;#R@8N1^KYg=fdKM5GuUF)=6&AQ_x|y<3A~^?5NP??rW9#GnsZGWcu37Tr;u-`$DkI2>OT5_`1{)i~m^Y zPxDX(CVN;xkf@l^0}}&@z#p?ZGR8G}o%cB^4hDHpPPgi2{lCU&_Mc=+xx17JK4(Po zVx6(hciQ%CccEJE=;$FEFf9nfKua~MKL1y$ppZaE$+#&gOG42NmQ3o5Jd7&UI# zZjK5qzTa*u76R((Hf^k(PxM}wqK5lsN(7Hu3TLw>K*^pZuA8KJW58UAa>>&aE8f5W ze3?T_QEfqIU_}#Ws3hZuQ44o474}FRo8~vwIU2vcjFhG5G~KdtHA_3HT3Qho);G)z3QmQ z?tQni?{s}*$mnA!IWa#h8++TSGmkF7+9McEm=U@%mAs_t{_=+1MbW6^cldHIMImMD zl3_6;xL$DGau2s&GEx`pmZ@z|xztsMP0Yl#I=SB!l2%-Y!rCI=hm56(#&LSxOvmn-n-4n z^*>t+LT3QJI|_`Quc7ETS`^eZ{E3x9*YdO)?Y-z}SCWiDHE3Fpcf5Fw@AnFxL5$-U z^Pfb)NRrjE^5v44V+6%t!KLsVB?VYvApDh~s~tk%HjAMZkV^96>5He&Un&g*eL_@f z+_L@Ee!d_-J6lvUc}{+Y|G&`x|APEXekSCCToC*J>VM>CGFOYNr{-JD=d&&$wK8^z z)Mp&nEOrSbClMZZgPQV-|6#s`I{y_Bg>K>b2~6RX+=1mVr#EPoK1-^aW@@k}U(BDo zqY*^WZRVB>#VE~8PoAGHI#Y)thV2xc?V71~W(}jJ#SXf3(*r{a&Wha~-31APkz3aM zmXQr#Z&+zZ+k!tF?lCv{wb+@Pg7IoW&a2~N2?ll4nsg$9@vyZe9iPM5ex}_mZV5|g z5W0fgEuY30FAnx;!pi+jQIOmrd!tuj~^~-4YsgW)N=?f$vmOlaI z;tr*1=Q)GSYs2kHfZc;%N|KL9U3&5H2s~AdLZd@0wb+PJv(7fnwa}$BK}sspHe&by zs@z$h-a4><)ijfe^sm9mR_?8c-^qvvw+p5y3X8rh$Qd4d2_8l@53efX*=OgT8Jggx z`F9|=n)gU9AhTXIqY2T4;wIobSNd#o{LbAQ)m+(vXEPIUr9CHPspG;Zc$n;ap_klq zOck#|Lt>){GjmO4cblH+3j!~OT(>aJANusMF=ut;CXk}P@-R^^Gaq2uFJ{&%_(rPV z|6>%7Z+W?&;l0#MIg&}^|Gn~9drSvU=O0hn*`7kggCP=bvs6Po0DH9ccrO|OMe?c! z0x}~7y92JM7RP&^1{O^x#oEktrsM{w=dLq`-9%5SjWwFw7bjnkIeB)fPI-aX2sY6c zO%#jvBE_hIDy2Z}h=lYRXuc;rbQL>Gfpas-J;koslL4i!qrCa9=b=j77(c_*BE{wA5-?PDmt| zu+86YF~pIU>?N6#zllPWFH!1MUCEY}Ifvo3Oz|Sg>XfBMJ+m0IRO0D5SSMZ{+InN# z$TB_NaHk*LJP~{u4WqT$4e%I_2GnlQ6<{H&W%CHs>I5u$_i3)NK=nm1^~SA3?z_b5 zgq;@<-75+_V!ms(%BU+REyBW~R`v=)IL2NA`zzlSfG*s00K-ST$9?cv`wowF-0|cn z>MrU^_ckLm6JUr`3{m}xrTC)P5ZT83cT`Cwvr#MFXx2HWZfivMf|!MfIQs4;F%|ES zWA45-FqVVPx!V+l(PY2qIX7qNo5+aVtM+|kB+_~4uv=(|UBDbreRa-O39y7}UxJpw zxNHIxc#75KGFwVj5-?9$=}Tb-2CAtb2q$KW3-UwT0%0`SVCXn@@E`7rqqg;EStmEF zGF~JE6W`VqS<;LaiMlBLD4S{+b67plIkpI+!n|xkjin5_v|>;>VDu);r`SD zGevGp!~4cK=mLLsYc-&(Xr{eHz12^JRBLN}N2JVXr^w6gsO{k1*O9&3QFs zxjFPc@7br~jOqJoTbFY|tQpn!)a~_9bf2#PokuWP93MB5+$JELqJX>IEWtT>`Wo*&a5U>WiKcB_!$Pri8i{OR*O1>f1It@vNS zf2~#C{QrD4U?=fJHt0EZZc3!#-s@9%0e18+PFuS^`Qqt|r_WE+oL`)(*;|(R7xU+* z!~+i6X;IIN$&PLQyfBH&qD*xRx!y!)vub8o$V2h<-0)kte%6h>6F_djrNArlBo`%n z;y(0EJ5JlV%k&*Y!ga?j&5b#5@Nbou9nJSi`@ma=#=`s2qq@A}&Hui5^Wps6oA*~| zZ(out`_r}rifr&zcwxnu5An5!;hh-3ih+jru}=2))F0!D#nZ0Tx(%L&p$f{z!Ncyi zqO2OI873FxvfirSh0^epqziRRu8AnIuF3WIN*SsF$DdS-!`guVY$qZ&p;m4y?+m801}i)5R~LyZ);W#;h%kq))_Rr=q%^=V?N(AQsVF;@~7vX$Pe>4-s)s zU=#@S`SsPAJ~sf?WlN3vFhN@5?JUq`z-nL4NQ~%~v6X7T3@k7c#LwxzG>6Ps)Q)u7 zRkJ~AJ&0%XmpP5v%anKuqK1@04Jd-T20dw$5_2)7Q=V|OA{2bwSXPgf7JV3hyRpVp ztRs}n5<#($9XTOtk&9x!AkR9>;y;F3tXouJRlz_6uj`-#%m^B`jwygP2C7G5D!bC^t{P8#kb(DC)*2&K#6mTBwD{_1OT{2E#!F zMm-<|acu3CxPZAkqq9a9VZKL#Q-w~C!qU? zwpRoujrHtIRKwXRXjz-+|5dd!&y4!=uz*dcM-!@P;+50U62V7_X~?K-#sVrH!-FHO zjiQgW{WeMlg)(3sgF@Lgy3dZY4+`a=P!0;^z;*_O^6~SW5QUQEYD<+es?;(>m)OuP z{aTeN{Jl@5(mFOKw|w~>*dGJub&fs`n{n!@n!t-XG|7;ibIVq{5gKbnJPJDBvel$R znsrFBrg?QM#hqghlXIOi?t$dn7P2jn+jh&e$1KnG3{%vDZebE?OUR<`-4G&~!lY8S zMDk0cAW5{16hi0En{fE7Z%vp9`j1Mq2mlM-9}n}+x1E@yGE!pWCtvMy8Bco@-^_~) ze4sf?1dTy1ZD7Y(L>$TT=JL?`sxYOJk;9wG`PU7o0)AF03iWpOHji0;!&CM>&tlP1 zzG4kXyo!@rh!4{JnNsdKE4K;4r<#oaW)e@7$myJE9mMQ=*+ASd6pMY~ zadlvEb*bc?7+l-qZ<#{Qty;p?*jt_IMJVRhmUvqZrdI3Z`e1F<@U&Wc?18gY$I@!; zu>)hPJDJs^R_+Z~t8)}xcv=sFrL{ATR`c+p?Eq}#>dDV({wPjKYPQOdHL8cU{)!0s(u5!3L zMWL33s_C9!et-_f<>xlgL>qeC)P&5}yLB2@gJ~M8(P?&4Y3en~_1eYughA#ELU#ci zL9;CaKOwCQV1l(;XB%})9) zy_XE9JJr+GKeINIUx?LrCfl=}n#hgM+fo2<2Lmu1IM>z!69DrHlwOH~+*8b%0Zua} zH+=bmTx7Dqf?8_rGo2dAv_3h(#PvaFToSde?IH?un}5m}y# zysR_otgaT`-X0;nOQhTZ;)6rtGe-v*qoFE7FMQsE@)f&Hoj=r65^uc<-5NIN<%|fd z>Uv}f&|zV^Qn-FT6nxBqic6Tge`s@gXfIFqbXIh&lmE_5@Cu1_P z`q7a)@A5fsk28^%1A)2-Cr`+s$YDeg%}B!N9kW?Ln$fJV>hjapR9X^PwTc8)t|HrD z_8!Nw6OTd%rvnr;~jHF0dK@xnSh- z+%otj%O#gevy7O_TQ8xk(AAXElKAj6rN4-bTzow-!+QFWh_)jhtfi1w@AkNk=r2Xi zv|bXV=#)>dD>wS@Af81GP(UuNUvfA40F(mFBlR^Tu-3PqBU)9B)xo@c=HplLgg`l%z$X_|jAKhuAHz zHyD&Gw!24PJb_D*bB1+_;ujX8rgI0vc5UqJH3AP_C=;~|54U`<}-;Tky`g9CDvs3R`uzW*dvM;tOMLX&*-Llwn zT<-(lUSIAct9@~5o3dIz-6*T!hY8E?c*Hs`48}(FaN>IV%@lvCo5@C;_mb$_;?gC@*Bd3NJLEn0febwm#Sp;yJe)F?$*y$ldgg+C{!3rEw_W`a5 z-2R9iw||Dqo4KYP;{qIC zU!0!5z8I(RZx`Zzq{OI@Y{ZpMG6rB128K3%E!R_YzYP5aGCxEFa^i9}9821RV_anzfWZ zYA5YAn>A6^{JHc8`xy>uVD+?Gs}3}h8WCPUK5awO-a=Bs9&-TwgM`rclr)`LOxxu! z>9ZQX>Gjik0}59}UeXqu~bKI;K@ww$>kc7PnJ<&7=v> z))h$eH{*mUeayb<*;acHne{@~eb9|(Yn}Y*O8%UT@av;xT+#pWIF9Q^IlqC3J(y9> zdg6MS=DG%porSo3cV0L00kpqIdG8OBn>&y*Ovl?^52^SvB+*EWI|)$5kXyMKgiD&! zlqr^1(1WG~rT*vJCCes?Mz>zC$e=npJ^fcz{0UY5d53+s?{2 zbxTrN7^JQXlz&FdPYiKgiX|X{w~_!cZG7QXA!pa{@P1BO=cPK!O7&l+T$K#R?+#=N zfsqmMUu!mz0jC3@?m(Z7tLm^FHmOKi+?Wzy`Q+r}1hG%Q{PIg8azaVWBA%{4=a%n+@fGkAhL@Jt_Gq3;d(*Xhu_G?wHmwJn#Xt%}HXf2n78cKbSyThEZ?O85k+fq7X3zgB>!X!1eE)O5J+8MKMKAgT` zsmK{;CD8&Xq6A4n?YB&1OkNwNjE9R(rAP{eori3O*L9v=X?-jK*gj;|zp!CXzhNk{~%bJS!9KYyi<^rdU3s5qs)d7}#>bQkHx0;cYOa&3RkU zEHl9m^#-{MwhgZfdQxdj5mV6Y!<*KxKH`vgXV=a@DYL zmN><(D{5(B8SDK937WtPSE9&`&&w8EibSsO+--@lRfDP@bA1KL9n5Y-!rM7ace~K& z%H`4?jqIJ1DZ66ftWdWB7k35NwWQKm6Twy91?Ys$SvI_4fp%0@eW;~fg{B4yLYK?L z>wFKQbtYnVRZJ5u7n9yb|E8BQ_^Q0Jci~Wz(%aBr`3-FINXXXI-w%y=t`Q)7~+ecB`n<4T5Tl%D{ z^i|cYAhH5$`*6HId4#7*<$PKw|Ca3oALx#ub1iG6TptnWHTvofY;0r*9684=)oZG6kvAECfX&#?mwdRl2 zRkG@NS1GlBCi?GmDhg$D7(5QIP4C#p^nx`R*i7a5bx)ReOEh}9eC9hMJEYhaPBpZY zd3zn4YJ*el({QR?qEk&v(u7MWs#D`PwhWcuO4rwKab9scRBzYQz8p=wOOp}i&{nIs zRvCOL3|RS|JRs0l#|5Tg&IeTr3F7CYbI|cXE=7k~4;v^uG|L1^MtOvw{PGBI@p`|4 zX1Syh^H$@kS}9lq%*)|~B2eAQs;41OuUBBIs*1v;BwDbP!arYXy0ui;YHC~(t;{Kg z#Y|Y9N+M=QwYa<|ch5TR?zZ)COA)(ae8unTZL<_!X)4#7rK%MlkP2V5$^i?<8Y%k# z(+x9F6Hu>a!K)aIZ9#Ux)FE5C5j*q`SH}oC-j)0CsIRBIFcyDuJbgt{q;9DR~P@}Pk;R(E7Dv1uU`*;U>E-k?BwawubP1z)>}>E^)W6EM)q&*60!1FFe?x9GnDSm5XD)2GC z6W0_qr_n9ml1g$Su!@ss`{ar7gQtvk*#glN4S^6MW}4o%BOFuB{EIHsfU|mNI(e$~ z#MhuEJoxgF;35G07xqf)I9_K|my`^bjlmLeE#c1&(?+T&0ryN(PG&|D5z!=|Tpp=w zF|c_q>{+#rK)r?Z7g75IvuGmy25-7tCOm?I_;6E&WiGeP4b=2~&(k6$h18o|Wab%f zuja&*1H8Aj{Gm7s&!EHl{k=;kn3R;POhJ%QYOj~FfXjxN0qVcr9>omY`=z=m%Zlh5 zhcV+x_-V8Stjk3xG82gQL5|O()OEJhaJq$?jd%-^Zcoa`>*CF+4DAHPwN7WcBGkrl zmE7Mtif>9|jS+j%Dl(N@cu=)?97b#~@VFu){Xf>&V?BIRHp76G_EV$#g8Q{ehlyFj z6lAclvTQ#_&!pl`Z_Xbx@~uQiYrA`9_JYu-wd_!d*Vl}(81rKiR(78knLHxXr#3{e zy<;&WnZ6*D=nZH^%9$oG)UnA$N@PZtav=a-GNMZwapl2r%xQG1Uj?5X4DX0UPfd8U zyKMCoAGOFuF>lx3X!&-OP-&^#*sr-IOA@a#+MT+KadnaluH*FTQo6T!JTs<1wRiG~$mVXnt?GnoagY(7>3euw&;$hm4!*U^v&kiP%jXL< zx@X|oMVVgl8aqh79-!KO$|MLHW$b)tjn`AWrz@zNXW}7iAtoIcYqKMpb^t;~!hwxw z2FfHD#r4w4c63M5Fh{)+oI>H{pHU=Anj(r`3$$XG{(e{OpBt7g6RN^vIi7s)Sj`>p zt3J~-i$^rk+wSVSv#v;i@}$;6v2H9|%}UfGt*G@nqXahqOQ$T3E9@CGrQByMx2NLj zyR#$0$82o>%iE>x#)<(Y<``pNrhLGLhbcJrW^bRGPPX57|EVXg-~8+7KOMA#zrNEF zC)W=IK!lh#+_>%%yzSkUh$l-HjSW7}JG>YZmZ`j>!q%0D9q@7-bI$%{qbFd=O1o`D z70WsUmTfj{CX~Uko`HKx)8r7Wg;@OHh$ta|?*K3K_Ia8S_MS_qgo3n%rJ)#XUMeX1 zlX@rE!Z-enD$Ex8T_)7;%B^pM2tnfzgx(2maji59$iQIVK?1YN(-ld%RP>gO$wVt0 zZs-E8WJaJyKZ{9`DejLx(r=dE2Qs)y%H@p6BGN)}nX{2qhq^X@6GJscgO{I?I>vT1~i~#Wuk`AT44M-8H<>dG+!M7rWsc@ zCK@AZ{bSWDkkSm%!*UjhOsc$yV5KOT=HixRWom+!1~)z>kl3gITp1g|3ZWZw#2AydQn=4qBsa8lzkJpb`S=02W8(`d$kSDEzhxG7Mq)gG6 zD%vYD`!y|*vrNPb5X^~$3Cr0mZvtY;U5IAj7Yj=xsHLRFOusLFt{hBeB{ zuOYK!F^LP%#w%vp6+cnOWh&M6wjHhYoIr{A#{=9 z#;%jR+c2$p_M9nM1!tl>(~Qw*FUE2M!VZ?9vsY@270nj|Ttq31T+-6FYw~?ubchG{ zTEt=gyfx^y>g6i=0+V|rn78nOAh$yby{L7JTe6^c+gq9|?R8Wl&A^aG@VU{m5SK_m zM4@dxJ#8WRs#BG$3K>_eowlS_*&#A=c@E&iq}xTFKs8|0sNGosY{v)}W4! zX7ALCH0f+rJgjS{3JYOS8>4b7;WW-9AflshOb0~LLOfX5(`(%5^y(5`w)et0{D+nH z!A@e^rcASgs$vol+WkiF5It=5GyHeJ?J$nF5~4k4`b)uG!c4~i6G*d_3DQ#wp2wpl z&D9ERG)Lv=gFAN9CIesU9TU<8CFHi+dIRaK$IQ4{%h&{1z2JFPtNc*249U^7tu{*+ zdc#^=9_cK@V>Y0A2lrGrb|36ap?3!x7~HX205z%e%3bs(&1z*m5Gr&Aqz0TW+$6YC z*9H57{@M}Y?G{?MQId&lWRmqu=Z(?3Z3Hpe6lfNv*%_q`M(cKedUa`-^kF74WPA8* znBs<(w8qN~c2jB}HlI9H!a8M{ta`+=Sjz(ti(tr*MHI2Xqh!p6E)sGj0t|Uls-gvx zM2ev8P6*GWqDfL=b(;+!K;tyNBv%m)W8qHry|I8U6KYIUvm%Qy)RC)IpCG-Bivn0R zBwFeC3_S^zbs>zU+KmL3xfM#$g+-o)i?XRE_)>&LG)`%Bdr$LNLV*LSI3l(kq1NUq zSlPir*b%LknP=;IraS4w>bJ0Cx~`)>Ep=bXN(``y2w|NTN~Bcr2rPhn=6KiA5~ae% zQiL5*P(?slGK*!-WV5y!xbN2q>Fh3C8ixj9t5nhnOCcJ4%nbMjLrB1rp0mKind(a& z1sL3!tqD=OOq~I5EM%CL@H_H7_|To6(7c)?;$9MUFG{NyYJXwNH8b2u9<5ymOut&a z)Jqw4;fR+#LU%{{tlzUF(f{w`*d3;C&+Pd;!|Z-8t!dIE*Br*VSS~{qCB4_5uFoFA zK=L~#AsEOe{rKP++%{(!%c%m#r>Wq{XM}DHwRRM?CWN~U(8+9% z?BNYuE+`E3lzeL-;7j>Z=4H3o2f~yfv>~8<06jOxL@*|Yr^#~Rd^u3Cl)R}-U9D5a z9d}TVXStB_jqSG03}UAcYGAO_9B$&nO}uG(!%e)YeGWHqhlg!F)lxgF8yB5PZ+(Et z8=*{~0%+Eh^s|AIjawQ8-g!?~XK&pq6XE(>rmFl>C-~V7$S)rVc33sJT`y{h9qLI71)k5B07g)AAmS?<&jz|{WIeVV|~WzO!nD8lGq(ijSb8irKnuD;;&64}*; zDnsZ^9*5tCF{GHPLe^`oL=e!hM;CXw_79 zaz?%FAsLly@Sazg(T;zdnrorWETVoq{PiYgV8bgEb5iCFPw0IlYE>7HQ<~8^8&-Z*>+cFPpCj=#P_~Mp%g_|d&RwUPcua8bHQmz!UVb0~r zfcTU+u)dz#3{Lyn*tF; zY7+K7Vk|a;|Mb{Rug9?jKv+$R+J3e#n#z21$`bN(!SWT+dZEn<360^2L?(G`a#xJE zF61WAS4{yo-Nk#Y8$-3Lt?t+Boz#mH0x5#GTuDY(W5r<6nhqQ4R@3eD&3X7esz(?j z*{~Fvb#IzPaE}DZSM~v%q6`}?t@Y%Vt-_6i!~|X&s9@VIW*pSu&3bg%UDCJS`fg5a zLGV>>YlE`FwE--eFrXl}VTpCKB#=R_$tJ=lPqwjpVY6$t0~yL?uO$KBvQz;SJ!KTOE%$Z5@O#_( zm>02i17OvkwwoqgL+*P??LTu+gkC|S(}%g)Zn37Dni~!nwT_f7HP`vSTDgD&^#6L? zV|gyecWeYdomb;UK2r3H%mVzL=Oyt#uW+pKAM%_oiJ#P>r4TETQO5 z>2K>_@1X%GD~L`Lr=QAMW-@0tjL<<{?r5lnk{e-3SiKm+I;6-B#Bc4&$=D4}V1aPA z1vA-FYn?iY%z?+!tl1_Avx~n~B^w3xDRPrAy%C-a_+J%DOE(^0)v7R!oXMrY5Zy*1 z^w3}y0BLQ*_5i#nu; z*KWFM=3GIai3P)zhA$oA+BL^;_Q{Sj5xSIIF+8?XQN@lB4{?@Q*py%)l!}&Zanz1>Ka{E2%iKKiWN!o?5WLpES7gV>m z1<-1xHUkRC*v4D+mG6L`iWzws1|#Y8$WvsRm^2_1;aQXvF~$!e3)Q?TgwJn5(yg?S z>Q$=)+I*KXv?!FpH!)sFtSI+v6jo=U*X#o|?~qIO2v*1QK)|;;ADOilv8V&GMs@S! zOFqpRIa|;yV~JCU8C_$oW2Q>cXo0eT63w>~rVb=o2&qU)qXo~(5}8?9PP-$2Vkufx zl@POaPiWc$gH3C;X9ezx$G(EBece^&1>n@y->*>j3>yH_;B^*+6FlqRwC!NI(SRllY#;-9_Fc zIn*6J_01%}wA%{_)^EJLzvpoeV!-2dBG{|1CIZn^AUtN7(uX_u>uHD3+f#DEu_Pb# z4e%`KlxJ0a$7%t(yg>R~Swp{M3X6t8(FdUxMDnyq6wR0@WU}&Zrw_RZ-NH3t@9`jn z3EA@Ty0C?`Egsng>%2DD0F^4^Ha;#TZ&@^ty=Rdrwz5o$Id3hE^+ECQD`@XRLXmI( zQaVWZyNZbIAnHF&HQ=TZo}R>P$+C`mULqqet9^V;9`ArwQOE%pKF|e|LCV! z!<}Z3j*lH2K_W8#TQ;DQ<81tcH8b_X1O%S>kes^u&UoVpADd6r+sY>HSboRZ{jtgT z8r^fX7@1_{V^AQEe+&QDX>Gtu)oC1?oK38kYN|7g9f;Q>NbNZy1&_b#vWoRmc{+_3 z+qerK8vcC-1$)C_gUqvQ77g=9qhvXjigs&8iv`1+StCttDLeZCV_f%exBQOVL{5?y zS=PR2L8=*5R5I0Nj;rfu)jOJdxM#%t1pX%$6WYNtOzl}R`m0g%vRW@Qk(Y$Aa6^9n zYr|{d+nUBXi#nRj!nn%iCR` zy|yqub$4FQ zY(h`n56)%zkpGfh){xUu$lZZ2OI#Lm9J9M)8PjJ+Fock7-TFMKBu}0_8IuW5c|!99 z3xbtZ(d=`6leoysfk05($rExYau`uWGmfGUd+zij-qJLMBXTR^ccD@&L`qWyV#Ba_!gs zqC{jSd_?zZtkrW4;XW)gAJIS8zT)YWkt+msEr;G}Xb8GZSqRkdOG1f3YYP*h^Qrnf zilk0MaVAPT4G}qlSym914PWTuEnv4QrtV-(*yfzynOcrJ+oG%C1TV33IPEp?`ql2^z|+`c6b9nU zj3E+fifA;8vgBD^6sP&f2;3-P{#XfM%^nc8$N<$JcJC<)2MTx<7OQt~J1~ zdLc&}h%IUBK^c*DTn_5^`r`Eb^~E@icVSU1ii zm2^~MDH1Vn#$fJ2h~Hu`@`#Q!g?GQ!s70R}japcOR5L5Jd9?`6Civ(R-?LJO`+kLlLW$Lk(Icvwmx0BUcD4(N?wRQ_8bS_QJ+9X+3K!C=Q$@?I&1LG<)eZTt^?y}$E?PT=^YP~G4FLCozj)`1 zjvk0-OAs;|jliUI_%E$9$H`OjJu}~j^ahplNRj)6C}C1zZrv*iM-}nj%-hGh2d#gi zc&zoqwfsY1J@g*$q+f5rp{=#O)C!#)ISzqJvV5Krxk?kB-But*GhvD(GNOO5`!{FQ zpI@*p9d_CQJ9B^4%gPJ}Wqr>6X4!-Z_HZxrPoBW+IyCl=JoA97{SG{r%SiGth6|6@ zxNSx*Cuft(x?g0=`00QEYVQTP%`eg>Okpv(Hao2HLg5YsG{4OBt)#1Pb3Oj~h-B>EjEUR#$$uKNBA|UFe|-@QUtU2KTEtgl zD~d#xv8W3)H*86M4rhD(=WNYVic|gXf+k45H~TR2)yW?|y4q`Ypw%+GDeWbN?1=WF znVFf&X(si$)%_(0#>sg=4pp@D`uFyp1Ctb)tFGWUZ*VzYl{qlJA??`-2)jDmqGg^h z*T)_I75qkCaS5S0U3m2HFqZ}v7Bsgjed*-OpJ*KCFHi>0j3$+LCX8Ds2AY*#>_fF$ z8myz6vn%LDl0X0H1T?unJpa>)ckw@L-brsu(g*b}?yS!Ak1ejn5JPPWtz}IxyjJs4 zbI4)ybufguk{aQ9+7KK2iO0s;^fqKn!5hzg3DUhcnH*+%;J^Jxs~3NBc$l}>{qVQF zT=e8Mi&sE5jBfE*B-O%b z<=ezPhM(HD^VPX*S*X)4TXWZCPbsBXn#=is^^igqh5d7li-1AN}) z*NmS@8B%*~eb;@|owbv}cVyYDI1H%s{;IP=c5 zCJ)B9OSb)eXO^g!D2?~N{oMJwc}BTsw&n~Ap+vStW=+$Jb*KFmJaN`tu&vq=7{6S) zk^pDe$V8D@r&8Uew8I(woYLy|Y0}^Is4n9kiER2|OY_TfqhZ=iG-);&3D{+1bEiGH zM2|1ew?3F6SN9_CQ>3H*%31`C`E5OkwEZ^nxX#`L&fXf{UfUV;YsBTHwdWD^`$0;- z+u9L)Tt{zvDcMn#p%k0m*=$vnVdwh?ZuuTO$G%_;8G7L=jzhs{<(#F5<|1}9TXT|J z{0mBjhA|QQG#c*_q$)}Ds))>m!7_yv+QaP;qA8_Ign=iqxXzxpM>W|N!jmiI%D3(d#`@l5ecw6YIjNIEkd+kXJi1T zZQVm;4zmR_GU!@1kQ{kI$6*L?^3}e1RrZR@U<$}ojE5#7o5QMg_(y~rY{cR+i+8o` z@YsZ7MnxE%eUlebeO+X65>Pm{Hrpp>8$DobuSw0YaMnTfhxHo08$)GWRrU9-g0$WI zSU|7Fbgs`6%?K@&7(p3&8*Du$KV}5#Djz}JAiM1-GScb`sVKu?^2CUTJF(DUHD5&VCdo~Ls%fjFg zOfs4#BJ!6Lz!r(rV2WfhftSdF(b%F-Yqe;(Wbv4gGkq9mJfbQH%uv4_v$)WDwaF1( zFe37pVR`%+iJT6N3<^A6APKl0;gqYK<||ZQ?AmE&h4=)46oyeDS|em%`PF^W8_*8+ z*`x=)|YR1yXZ<%3KJ~Yn0Jwk_l@i@0d z$%0b=>5WO=$>J5<9N-LaJvpm<6Eq!I{a;FE?h_hLpfC=MwDPydp2{H=p5p>MVi2$}X zICIwQpJD!?dfX;Lb;6|{?k!u%%~^!X6Fb)`bzC106Y5c9&Nuqd6L7U4gZ6x!p>~8{)1^d=00dMtylIs&Y=uq3wxV>tm=y*hP*vo2@AOd5;e_UhA(sVfyXiOzT@ zN~@n=tcvLBh}!DatuMlBEsCpGzn+M%5HN7tLPNkdrWF(tw%)?;Vc6=?p-$b=ymm9| zw41a6~Chk1_4Ya{S_ZXT+2U@4M9l_gNOmYmpNV%C6&m8@--l{D==Z6gxHg|K%xS8KeJb_jlYXE+y>X5US0OICOB`l*b|&5K3R*xC0Z1m z048!?P3yZp>QC^Ee*4SQ#?W4BQCY=t1d9!KL6tBm3zhJ#z(K>ELMDG07|jD-$e{m@ zVn^$(&fTxA^fOjfw8n&w8k?}?W#Sj$j3YG5gu-B$BQ(YLawL*6jM&Y~&YkSDUb^G9 zu~wCQ*ePhm^kJzaqXkQ8Z#_%5mwIKcjCxr`G84t}R1z`kAN=8FtUbAV78q?=wXFjr zW_Bt&k@m_dAdNOlxMC-LFF(Yo22fhiG_$|MwfjJ1*)IB{BxP?!xo0@X?!IVjyIEu! zEHT!)(rcA`mY5@}_Vm46`7ANky3()DJ*2j6^cj@u)~m}dOAM@4uO7QCF|Ku`SJxg( zOmM#3W}iX%bGPwx0ec2Itpj zf?J7Jo00u;XN9>^lgq0A0adovTM0>G^yMmJJmD!f*`?#MQ?ON|F}*>w5s8XCXPHV?A6PRl=@%YCQnmVmmdJ)^AuDyT-umOLM52{P zl{rTLm1TG2j^^I_O-*fU+P#2hY9#UzF6j%1fb7+sCHHRUx)S{rJWIa|0)b0{X7?ZH;BI&Q>qO;m8{bE{ z{q=XZJQf#9kIu2|OHUg!zSLq~Hb2Hxg`_+_I?M9Mh*iT9?t#4yfoh*{pc<}p%`^b< zXMM&R5T%CkXMvrDMyY{|J9z64QEJ_7h*BG()P^XvK79^RYD1LT5T!OmsfDy5N^OWz z+e4HZUfLRB)dDAH@VTl$wST8LHK{jP#7?6~6q&jaw=5t4WzCkb(E~tCt=kqtYzbg} z*zE3ML)d5$bA659VvpHZU2w-NW^wsL;1)C7MCsX%$tlTM#FyNZNuc<5+;yh#aVWp# zO0r~zS`p2vvdOt~DK?kG9fpkR5O9=xE*X&vQ6w?R*}sfGI+Wq#{BNAm)S`xf#N`zF z4U{gq!Jc^tKE>WIc@CR>!XlBy+Ze7k`lwVUaEAq(SmGitQ`PTi!eg#hWkeVrFMapu ztju0DyD??8=7cVnxuDU4NoV9oJyzS4urG_-CdHg*cqxFMwU(HyNG=pr!8>Skitetm zpVS`->=9Kqm9GGvMVR3@bl}cG2sbT>B(~6=e?HOo-^iSHEn@&|ObL>N#|>!dG;>E) z`^57v$U+pk9G~~|DJemo!JBAOviEC65=NyW&rV3nvqFWIC~&SUfBK@wF$}+~Y+2O6 zy`aFq%UeotQT(>q0E8H99QQtoHIhvgVprT}LEBIwzY=CmY#B+``1r|5(sH0qE-1{+Pw z#hfM;6`a_YT{ov<8O^-D(>jVSJWj-YhEQ#c%=sP5)^;TcFd2PKqg$56kz)1P`!-+qwGCRmkE>O+dqS?&77&@d>S8LU!0!5 zz8I(RE+CUVL9`D12zE7aXTPk|q*#f9+*7QxXprx2`0@q0$YcQ!J5;TGrc+M?RuYDT zM=lI;{Tk&5K*QS=S~j2h@7ZWVU5I-U3vv%32Y34J23~z>ttrl8&T`==btKYdE>bS7 zSTS&JJ7aM2l+Jr>M|iw6I>6b)KbcBpVHWOQr|q{3aX(UGR7f`B-q^RG3E#361Of(6 z*}PD@`zZlNUiiCKHm<)u|5kR{jU2W5n5pPEV#~#`>3w@JL{dLi@E1H)&c?~79ToK+ z$u`Otxpr6@i%>cg+{`vb?na;P!LferTs`Uw3SfqqP^G>omk| z=UnV{Up#;dV~s(=Up5#djZ@x{AN?p;WcQT}^s3fhtx(^zKi8;4GV0!x?VJY|4qaF01PhJm_GP%g;H1T^{>l?a!g}hAYoG|#08%#;+;Uiuou9}Rf(|} z#Mth$17#D!X|)YwVb^7a{0 z=&CJ2%iHl z67Q)>)sRiq?T1{eLbYr5)Rx&&q1jPk0fwF4g@z3DOK%3c%`NYsXHedG&s^-*Z1e6s za(};Wj`?1*$9v?AyV>C0K0`9aH*>%{m#Zy5yd(45&;ROK;rl@)*%pf=z|qz-qdPED zJ3Of)C@Xtkx!66zZo~4fgR`xBRHF3eShwU&2XwnvVT|noL^8HMf*G@&L+IL7E3%&B z9MEGQSsV&Kw`YlrZKo{ee!0tik^sUI0D`lV`xMjk<|J?C83**(kcr%xT^!JDX$a_* zQyk3b!g33`vxWmQ5B8Q!5SD4$GrzRT9QA8A{oxP#7}I+hG7hSD8qG7dZd9jwVd62X zlIlum9{9kwV-mJcyKM;gRne+yG}-CiPS%{L<)5vhZG5EdPSnA7x)0y!KBGxLo)<6v zJiU0bR$J2=QM7{P4^rBSjQ?D)kRoZclFEDZ zl|X-6&Ds)BwU=b`H}O z(@{9P66xD<@ve?+SRu318;Uu zYg{8iFX_AS<^zyn51E@*esw`pHAv2@^@0@`9AInXNXtS(q~~24-|jY%?x28b)Y~yw z-xo%(77agm`)s!f-P7buq?#Q!!?D$t1kx?nUWiI|ha?@wPn zZ5S$w$4N$)%hj0tn33;KU)B2K(gcFVjt3o-fU8`jOf6W^BsTy0U*!4t`N{Z1_jzrt z-_2&aJ zC+WM8483_+WkYM+P&*0d3`JC^7@KTgFjJW1$SejlP|rq2qft zJ=Q%Q{|(RAaqBwnW4paKM?x#W=&|kNDz{thA0vCRbU!20(Y}rqIR%$0i>q=WnUEQu z%^r^E%`@_Tj%9^6pj6BKCw`TDMBa#rp4eWN_N5jRyDTbnu(_JS%pIG5HPIjxvkb#_ z^jQ-*pLb==?Y6eHi9P(Hp0ljWaX8US1@qp^(*I*JsZxKoTKj)YCLq7;TtW6;6I+y7zQz?Mo&vBgRhIopJplZ>_-vM+=(G{lO}fYt zmGk+Wz_^V_YP}potLUE`e5(vtX+n#QjwkNI3AG@HZyd3ifO zj`Gz~iDQ+>*dJGdK-gY?zTZM9|UAqr-NQT=;ec6KCqqkHt6M_49^MC%R$XLE!4svR9Qa`6VR;4 z5(6w8b*-+5jUnCIr+W%=Rt^Be;5d_G@)8nY!F8x8PoQi^^_=MTdIspvXSJEP@B4uL z9u@f=yQ&Jlg_7|iN%X%X?7(;!+`w)EK?&sI?EG&RzS+_&mJ52zj`R!5av((HG0Vs9 zJq15hzo2+}Zo9e!5mWt0(jj|0JLf-y)aZX*y=`ra&FAR~BfyvUA`sVaL0 zUSG{rtIA1!Nq6&V^6U>?4s-7|NqN?D0_ZN@;{=}XGyxP%s6CwZw+NnzOywe3tI&wL z2y#FZ*9)Tkz&D*#NM(kTuNfAPNR7iE$hC)kwj!q+EX|r1tjggkJFGzutuI|$WWIG8Hb|v8SBqjgjzsDRsAE$SRjrPviI^Uz zRClb!mCE_FP$HKYC3P(Md_?nT!4->Ck+WmEy7+;%dq>O{a&z z|ClWkvC`XkbF=zmVSIrudMLc*hOnVcMMbU^4%if|VNdvm*s8y-Gc6pBn^cF{Kh7s} zhV?LISNYU3!>JnN5jjm*u2M)2%CXEW&4j!z5|)=eH}Qim9W|j6qt#&oH#{(xaPu{r z90X9pX3F|06Ij;Hs?d_}u2LcGuv_(2yLWWgBM;bVr^HH`lpf5g+NA$h&!{%9+Ooo6 zy@SiF>@@zD8~8vXkyJ^`wi$QM#B#rHSZ0cT2Tkhj#%t@lc$_jtW2)$QUOv;loEG)1 zoFciW1|O2@`);um8D9T6c($~FRhJhI3}UtVQ>M>$&d^1@B-H7sGv*EhSs52kb-<#|c9KHDM8uPe4>ZwEA65sFd1HruDj zVwNWWo{!?PNyr+r0a(d!b(bvHhmOT#Vv6A;OtG@_vV3UySPLsGR=>|0LG6Fk08 z$sKg=cRTU^3K9=vGU`0(A5IZ#KJC_c-+bz~op|#Z?>gNhqmZxtvGK5XA7%d`Zalu5 z!yW-)e$)CEo{Se@j>kV>vib@r#}lKFLX{k1?Fm**EuYonu`{xd z>b1ioSv>}?uRmz|bRRYMWAcCh-*5io->gLonk#u6 zn#ddfc6T!V^7;70g6uXuGm7%~*g)FyY(9=e&V8{ibsJzaVtU*?Yw@ zNI57$e4qX1mS^z`a#lzs(rd#>Ad3}@Hovh3OwCScmI(}`F{~y+$7aIv(VS&ttussu zp2RGN&h4k~PR7svFviqhJDqVg=-RVLh~Joa46;78MgsgsvnV~Hc`l0O3)1`$c4{yALHble@47H$?6q3-_l6%|adU56CPhw@jX5COUWgp(gV(;F+rA~w z=0!sD^_PD`u&U>as?4L7XEu{W_cbz#k8fdZSdtPOi0~fWBbB z%<#I%Wo7nnvha%;pml)29TW8LOAA*9-pm{{RWZ-_FJ<2)Fls^-+prO6g(c4{D8^BV zDw8w)6;fnQe^_j!F=}Yel|7`DQ+7*7Y^fj|G$<(>w`JJyo)yr09s)5e@BtPZC<$!? z&#o^fH>KsONtr~tJC2!Z2{gf=c%%VX$wdl%vMgQ-o+&3>V*Nf+OjE8TC|gWI-S06u z1I`7}iKw)V$z?{)Xv&f^+EPw`p9`QVEk~NLVN2jL-Svkz!wC%YYgsQ0wz4TkVpi(r zwF8WY30+HT_3-L?d%I;0m~Yn|#1t^n<8bYpdIXL5AYs00&87f>1@~oQ6Y%sNfMVsz ziZH!R;i6OWv1Zfa25;NO0}Wi@u01NQZsjr$lbGRsj7hQ(_e9JT%M5?D5v_Wbt=1Qs zvud8LI0sZ|4)uck@8QX>hlf9$jQ;Y^-#=YF{i^);z4Pzb`S-&4ck29Gx__5Xzk2%r zZEK%Pc*?^-Ctv9&5m}?M=jvsXfDu#M+cayBP?8Lw0I4J0;GKwOdXeKIVq^0$E}fK4 zK#B21MuI!l+2c?{@(<5oe_@;h!9a(<{uenJKmWpS)AZTpGQsr9nY7HK)i<&EZ{o1@eF)!P4Md)0auH3kh_Op@2 zrNfKQGbj~4UAZe{^^{hDYy+kgd~U>r`Iilr;+YNaO_w6m675)%SxW#zh>;~kD==ww zIKSdg59w6o>eqXkXHUP{`cMG_2|QHDILqUsiB>-ttqEHlQStSKHF~A~)0EXutjO2C z*xDr_6IKs;s>k~OVEg>%pBzeIr*U+ z|IQySdoo}CZTW{0{J}R8{Xlz!cUYJ|OuR>Ax65JRT?ZD=gzbYh%G)Ec;FsXD78&&T z4rl%jw&XkL`(2qSC0R#?Z(uC@#s#7lSt$P*4ql zfl5Hzjsd_Gty8mEA8==u7kj^K@QAyOW~;Ye5E~n85e82;+LDx-Q~dWU8?|f_?yb7y zCq>O4!ZS4z`3OJnm|~mTtSMfgb&RJ*g~=gQNAj5EfcXp;Es-Ji&1 z_44=8WVxW*d;Q$GluEIKa#%yLz(M|mqpX7|XzgM?&)FP|?KH}Tlq4+@#g_)~$za#< z%=ppVTt?UYF&;dIgU7H&{lR1SAv}f-BbY2%WR`B-kLbqv)sF2NeK9fs8AjXrCjN}z zLM=qje`(48s~b?%zG7 z9B_94tuh5B+pa7N<_-X?xwd`Ksp{+birzxOsS(f(P%Q~!~3kR`Y>~*zdV>n7BfK)^!8}Ups z$e6rqH8cH~ksqIYO(lMGo)LUnwTy4|6W@;f?EoyZJ`CHhVYnwKO z4vq-pzzN=BUAx=>yvCTW)n4F$)aaurGETX^Acz^>msvI{C?e*3P|scn{~^b~T)Pw+-ihIQJjo+H^BBV4rD9pM8ZT6uG*t4l?R&gj#)ZC43L0x26~Bu2_)Ic&KQf4~L438gL437->w(0V z-|&U)Va#7mUM29Z$7<9% zG4S2*$7(baNh0pA8Ad*aRqi>nu^Q!J(KD%XTCh+2;~(C?Z=FVNzuI9r+t*x6(@$^# zE0a7%bA~mbs%sawRzGr~chNHEvSVbPt(1wYT)YpfQ}yJXdHNlu%0wbf=}59A%_-Ij z%Q2FODiV3Pijm~x@=6|&v&-{qVlys}NGwwQ6VMBLGD&AsM%eo(DPj+6gKVg!9wE=> zV~AJE;(C`6&C2gXU=)^B^OT7BF(a4C@g?|M_=`V`PsVLG+5q~;>@{0YXsTMj zG7tfBs=m=g%*vc9&Bc!~NPkIr4#|8oo&XaRG1fk9x5AP+zhjy80{Hjio@4QZR`J3( zVHqWZA-2!QC%#mzss-7MG2*$;P20ZW*L*!>*A02%Yr0&A#D+OOw{WWml3bqeMg5KJ z2YYEbjl+a}XCiL8aQAF8c?w-RZ|H1Dm_olXAFI@tYJAs#NFPO#1S;bj!+LZIIoU!f z59%~TaVLFg9;(2T5GB6a@~FoLP0Mka2kP>qhdxgNb$Sw@*OPGFo@{MTAW(tDmw{p1 znloidb1G&0hr9O7$U1+hz6^Xh~q zMQt#P`W_t2qPx*=FpIW+>w(-b)^~hGbLy*Cy-lIXz-3=!XY~=iTTiJ`C zJ6l=IKchD^Xh`jfK$vdRso6Y`UbBT(6L{>qHJZn)&pd!O6G*o(P3B?rm_RKiOos`n zJM7b6f`sWdx=Y>VbhW`r$|cb3tGpCp2zmRyfzyWygJw?pmAC+R)!ExOZ!XSmN+05Q z&EwPBdeO5S!4t+EDMD@HQg} z&F4^&y;u^iNXq96gT^u!A|^a$VEg4+lrt)^=w?S)^pr)^9*u}9nus};cZe1=n=?X_ zq%2Y297WLDb-(C>$6Or(>b`~rHNZZH%Dk+c_Y=Jb+jgqIh?VZn`{QKmIl(CmQ>?9W zhLH~@mkTgVwDP$SQb8P|$jRlE2~P2IQ<|0X&5ciZ$TAbgWgixRbLyZD;%necF-O=` zki&Be_2qo$_Hsr<%9UdA5uB#7NBusQU1)u~;{qI;BeiVS z7{T2$T^TZ(dzhkgB*njkmcdX*x_!dn&6*>$$R2~5eC0`KQyOc-98aA+ZFdh$4h$o^ zM43RQ1#B7cO4z_2V9ojiCh^W!d*`ukq(q|kIBXZanvk=$oL3SWztS3nwDG+&-Pg?I z#b7F@GBAbr3zm^Pmh;&PYD)q$^?N9)&b*%3Av~p{4dE$+|JP|lc*+o-veAa{6z^0A z_O#xH@D#UwV!SCVJf#Eswyq{&VPi*ULz61tl%Vn#3%CozgQNY|suzk>cF1X+o0aL?GxioL7(86WKof`$t_=g2X?t&W$&Cz$QDLMRa5%le0S<} zVYI(xtDeEpoBXwcy#ro0<4KrG7r=vu@aG}?c{lnE;m@t#dZ6%UbUVE1Q-|hTCQtWU{`)3aCP#=O^BuR`zpc30PeK`$ky=w|VX0ey%J=tI5f zHJ}geH}0!#s#!6G)U z9k5Ebw*yJebFoaAlH~O2a!hU(TsF+3wZ={b!$dBx?!K^qOJusTQ5#44-?qkz-s2dL zhWIXxeO4$;doH+bwuaC%-xZ>hyWP|M*MnZ9V*Q4(!~9ik)#_B2FEK>Nga(@>+$X@iCB@v^zW z!uEMt*oGQyZQ~wlv^6|7)M&%@$<$~Q?RhmDG8W8zTCAeeIb|y6kwmh32f#DM<}Df4 zLm9TA4BJqKZ79Pwlws>bub~W^(}pr^t<(7A%dlPJJD@`7m-LKtHp&@|fh0VWie{0& ze$HdAPS(GOPW{-?{V$eZ{$#MXK*$S*8I=Mnm;%j-8MKI+Xg#0 z8wd6@I2)mTV!UanxHG7C?$`(Z`5E!gLA`U^px(6&c2MsI_B5z>&^|HVG*sLfD(+w# zg4z6S2xj}dg4sUhiaXm@#>ixFL5w#OZyk&`lg%O#4Fla%`HN~WSoK004?g&hC=@y{ zgL2ao8)=J(B{XMb$#O1Y9zjIF&jrgluCASrGhOeZY9?*z!E&j)m@*VRkz%m{m!UBx zR30-G9p_AnlIHYd1q3qum92d)0-_p*!&f)+N9BzVw^?KL&f;T*>j&5zc_pT0mZm8n6BbgC@W9D zZ(a^PY#Zw%u~Nq}=Xd(jd3jYUAdL1O!!P$*Q)GOUoV{URQT=7 z)7Ooiy5)ypBw(3X&^wHqW6|>$=Rlt+yR@^>2R!9c-!qz7h@+pZk!-&@1B;&TRv{~j zn3R8Szwp-e2K%&LHK^`4HV^Ev7FN}IlZGi0sEur&HiM+I}=6Lz7PwRv+5}5 z%|E91a!gbDi^#^2NRLmypIn?hw+!?2>hc;7Q^fRt{ItQo#|xFV<=+s3L;=1Qi>OFh zhJ|`|8UuC;A8tD|9k;4=A#H|fIB`9~RNqjL)_#jtT|~D&7-l%_HZBI**+6^ku+au& z!S%4~g=E5oq*Q1Ucz(u-b=;f1{9ymL;+Etvh|rhTAdf0m`^C6ay^at98&oL0u^5Fa zRyLC(wzVQF<*c3FMbZHx`+@J)`C43i>&U4x_ABRt1!c1Tt)k#_HV>_;dSg^Yd}Zqo ziiDkBzX{V+zn_q6kuY+4{pN_!j8M%CGphI1Y0QK6%4|?F*vMe&o$(e{bhWjpK9o{JxfpJ1#?U`4OTOFAsKD}Fn-ys8&+MjryN5Mkpo3jg_%=zZ*;sc7DC(dZxJ=Nrk z`R43OkFIeww)Vx4wB?X&<%`*`n43p_DF^xVdb*8khR3QrRNn|OPZ&8%L=n4wp=SAL z6)qI`rG;aPAxGc5ef!O;i=R%fu3lZ9o!(r&ee=`V^~L$co14qiSIrKyrdkVidPRC< z<2eiytd5Re%d$Do7!pwxO5}Rv$8OqAHXkW5f<8vegsPdy(>9Wwk|kIxelED=3InI` z((LX=Zt9iO%|On^j`610MrPfe6^J}DJF4P=jXlv0j)l-c98@ zyof=O3O(*--(Bb8H7*p$_?EIyN?<^dv$xHNj{g(?G16DdzaIamLyeK%SNhLmEhhik z=$H@C;Z%EdrY2v&gSrC9WR)rU9)%W{wnr|_P^O-Xq%jGCVY3^Zif?qb`ep9se&P3yTBO^f#*9xJ?P+?kiQ7CY4S+FKwC6;WT83Vcpz4?Bjs(?=KHMeA z9gZ`aO5+M2(8Jp0bWi^tpoV)dTQz_Z`ldQ$nGIQH9vTi=X3z#u!jA+>IMrj;k|xaB z@U?v$4tSn6iXkg*yzZ7pG1SC68e}(miFh{Oh^sao$+kls!sVA#%6pObL4~ug_)+0H ztZXiIY==!kYykf7Jvd0Q-RL(+vDR-rkQ6H<%lm~+y|b_Ejo;V~2YzGPz;6bA(=?+` zp5JtKJ-{q~T$tqnre%O>8DLrln3e&ir4PLZm=>oEFf9)wyB`ru%Yz0pn1Uz^R$AtO zuFKds^4mFu@y33tp$`K>O!IdKgqV&tln@_k^kN%Ih*#~Al<6Hxhz}*i2lj+*C?Rgz zBjHU0Ld>Avxnm#r=V!z}2ldWvgL>CC*g?G;*wdiiLHopb(|`~&AjDuBqLBP;h(h|D zqL4mh`D2VPz8CHd&mQctZQHhO+qUgJwr$(CZQJH|Y(Dw_o|AK4+;3L8+SRK~(_BeE zwGGA)iWC?-X`S9#_UpUz5DlYfyIsONy*|Ly1v)a0V8Bz#X0phuuC&WXPx1FpXbH>p$YJ}>;d z2N{Rqwxjn!BTz6Wf>XjnF7Ticp_4q_Ds_f6^W;n+)d$(yNzX0_aT3Z&+EfZ+z*R-a!A-s@Ea2C zeAX65`sx!$^09{d5}v5a6xqp}#(LOhFXMoy6jboE;DDgA5YF}1#(taWx6j<_x7X@> zc6``Id^xdaXIGyKTIM6ag4Yk}&OP&^&X0?>2<$O~P(lpda5QV`nWX(QAc$>b6yaoGZ!h=p+v(ES;qooH9nMzcEl&n55M=@or>bq z{ElAJBuhQpir_4Co#a+p`~fC6S3>DPZ%Q!uAk$b~u%o(ZVsBk@51zUfu*JeVz{cgN zX9FH=5->_&SRWM-!vNkmpXKlXqhqE3BfLsw=AD^*#;R|ENCt_6^qzb6WQy=vl?^pv z8~=3K5Nzp5R}OQ+!EeI>*K7W4Xm2yX_2A>H;FcPDp7Og24vYaxA@nBSVg)tTeQ5w$ zSO4qxNY%v>7;B1t#*T9^&*2`Q{Mi15aJOd3x}Z`cIu3AtMVcWv1N z>j`Q1IqadVu}Mi`zzgzQ;}l-!i8@SYI#Y#=wGSAuxbk*j*t`+z8?e~=r~!9E+MpPq z&Q$K3sL+3wM(Vw`zOLpgbiV>?s|U>xZGN=3AcfGSlcnUHKO*>g+PYHL<3kg1z>F}3 z4>}p(_9@VlpK;LqstAU?d8#~Mu_+I-r@~*2hd6pzPAAS%8Th}X2S_%7l7-BTh9wx) znsdKMOu!EuvPgm*sgHHTg=aryx+t-5ON%+|uvOS`tDqgAuDnjN)5xhxTj#zlds$KCpSupPlKwG!O55kRno1H$D>DW9Y)aooCgHm)E#G?-WBVBI z9Dx~9@)K2F%i}C@rf%dbj*<0BuqDnjw!wGEzx+b^yU;4%g75Gt^2yjH?}F?0sk4@# ze>>-X$NmB1u{sBOt?Z}vkJf@X)S}934p3O@mW5>nr8QUrObbY@(TZzvSIOErKv3$m z@iAWOJlQ&_g+ZO-E@KIfh3Vaxn_}kz?1j!B&r!3yE7xk=&|mKy zMs7AXvcIM4-zT%*?P9rRLV2l=bQfT2)-A{q`$l57;h;PbCYDYGuT+*kOv46u3=vaK zJl;RjkC7v|BsYAPCJt|@UxXH1#RM+&Y%LN6wBc&?_c}r@3V=%&(n2mS9|a|xb7!J_ zmLh`ssft*o$2G;_aV990I^pT2fn9Z3_4K*rwTV_Fo~FiiU*U?to_Da;ieIs-TFY$kY1&*6xmI5Oplo7<*0ADA=SJU5>sZF=aP&GC7s(vuLH3IwQf5* z^?o)f0C`v=0cq2YZ?))vN#_8%+K)oo9TWG3?UC}VF9y&`Jfll zrGxO@mx;G>GJs7q{ebq*v~b90GxL)R*>kZBY5lBEF9y<#b2krCX)3ke1wz>Sk2@0~ z(L~Jbd?)Uu{f5>_kSqQ*`TP6$WD{wZ2c6sZb_;Bt`t%x{{JWd-lqSz|VRkc5&FuU= zA^dG2@H;F&ueyKh9_(Vw`h?&2`pt;i{Tnyu>#t+fq2qZMxM$embMnr8eZdzU;kRGn zx1VzMCx0IoDE=-M*utX>-ooRI!TGZ#PwXyYFehM_+z*1UAWZP-^u;a&eV6>n<-;;r ze6kALYk3S+KB_u=NENS}30@ciyO-MD#OpC-O>Ug=#PGEu)3Z}lU0Ye$p#*Rgad+MvytM9RR!RID3Pb=8NBFPyhq%6D%>}BOP`)4*%CxAi9 zp&oM1UP=Q>@1#r#S;si@bh^%)lr$4Gax_`1(^d9-XP}&QtjSz>0~vs$3aJ(>XjEgp ze%GG{jQQ;IU4<9{^3W2S>7s_`+$_^dD`8T~o$Kix#==U~|5d^cnlgik*4P~0(LUas zS)^B}HM|w0dsB;+0LW>C|Mw#84#u1@AFZUnbZ*Et9RlUnb@tLTh&{|2=43fBTbHM+ zlOIhX8qDJeIVp#bdY)^N@nb)rL}W4;0`I3Gs`C|0{>zp3)o1VDz@IJq^YW6${}Q4H z!vOIVq!v^9y%+{=-w5r2@_?DTEsLOOQr~fXkdM^tB+4@#NT_qBN+$-+K_LBXx0w#i zl=``yt1{iUC$FfiD1Z5JC8<2LjAgD1!irU73pey&Uw|Mwl32d*`L-T zgHA4aGE`snA6dn;s0>prR!sVy4<{%5H9Fd9x(H#K#!&|R>o#6=gJoOL+|-AJ4r!`J zcc*{{L{d#Vvsy_~4ps?}z$V7Am{ql!;Bg0p{ntY%chv;y?L2`UIv2x=Y0{tnT@Syo z6_g!oYQk0UkC`^%sh)MsxvyMtbsahr+ob(o-5?wme#+PRx&p=cLy@FELRpmTP)m~WLR!CZl&!2L zYV;QEt<`+`jB7)i5FVQ54DQ>&yj71gQY+*<;%zq4nf~+oP}H6Z-j!0MQ#b@A?ITfp z+|!j%bV@zy`-M?^MA7CLJ;jlIU%IFIML~&v?fTD5WyJJ~udZjGtthG))XXwbn_$Ep z+MGJ1Lz;-@PUuFT@m*lpBggji~+qthA zt(fPNJ4)KB4`38tbb3vkxEI@7{{ zid6me(MG?Wbw;WzU41fHo62SN`aU|VSM>W5dexU(V4gKec%B1`_8x*(tkelJ=ui$c zrx=?~x0$q>yepml28W!M zPkEoiy1K^fOka!s(mMfKfttu8Xen;JYw7(!h9P4f+fr7a(nuyq)hjF<_o8Pu9Lr&q z7r6vYySd2O=pke{$GYXq>`c6j75bLDi49-FTSCJ;OGE#OYSWDKJ14Po8KW&eQ8n!Y zu;cII7-%OyB{q$D-^j7>*pZBJY>f+qz4n@%#Ot+uW>S(Ie?~L~1iJ9_l=m#{KjOjoW z<^3AK&k(!nrEfaEK=Iz}lB_Z5y}v7wn_+m7vU|c#qyNYdhC}NGF|x0`F*;9UXz_%i z#Wi?az|hwEy2H@YZDD9#<6e&EZrZ}o*0#k(v&Uo@i#^xwz9#tp#J-)C@2+f>?>@V_ zsoaXauvEIw^D*tRn8ji;kEOGj#oo6uI!~u_nsxc6XBZ>w&syNrUVuvu-|)GDJ4F4U zG~N4dalEuTce_W@^je|gsM60M>8r+CMAB*CHk=uFx!EC_5K6vTiDlp;&7Gmb)B~gF z0fk_-BR~gY5L(_MLRX6t=pwYcb^SK4$q0GMBd)J+#^?T9D4*}XIiFBdNxTSm^!agk zUQ~x(<%Sj3ke0!mS(n^2znG49I#iT$Qx)^U*z_PJ8P6(wUj#3*xmKNkn3b2PSXaz| zpoS(S9hjT!ObPrafl~^~Q*Kj2Ex)lcF#txL1@?zqB9utxb)+LOgc4sVu~F5l(K z*<+S}G_CdKD2T37g^E`BDbIIxQ>Pp-Wddp;%NYIe0?=-m+*2c*vj3ehw|{V(GUx;8_+fKK(n`UJ{&BS(|dDiy+{}+g1JI}QI z@T>#d+wm|?Y}8eePj$K9$llr4Y2)U&qSw!E|9;BU(pO|GrX(Hn#m4p&qtNp|cyJL?SfRP(6*|9E2;cTz4 zmC-;KL(@^L4rhilmPT4^qf9i_Mb{0|w7hYSFiAMmp)l5}0q9R6pjU8CceuK`=qbku zaQ~~Oh%AaWsHj9i9t)**$Iy^=wkorz29JeOjy$s2hnlCkJe}`4d zGlZ|+;vCT1cWv|gwnjl(DFS>-t9}{bh4w)7X+L+UZ@q>K0JJ^IWql$)F?$HKD*?@svZLN-frGS8X)+i4f_zjz_`A7Esh!BBX`9 zbp2b2f6EJ0$v!~u{@xs#`?MTRQ|&iSnd6TAx%{qhEgLnniNk8$vQxG1qZFI*Tj%Yt zIPC!*HG+_dYNxc>%OV+6S_ETuAkQ{6)j~3l1wS|tmiz0V5SQpa;>Uc+^|*Mm$wW`P z-*z4^%X}^5WsJ}W%W10E^s&`qdo^)+fw6t03*+9Mw7z9BoO4|2)i@w>ty(-1 zw}bc1bvQABFg|Ladz4LRwD`p~Jltw?mwVc~3>}x?LYQqw2;&hOKCngkK?hSG4Pp#l zunr$l5*7R%D!7DC7CF%@4kRN!tBs@K z&IB;V-3%F1xP|9qU=ED+L%2m|yzx2&0B!s<1I6Vc7^nz{^*$~<<|zKbM2%PcgNZp% z{_nc{-|*ksTWS+tmb3td=N*vc?`-PnmTsIY#=`EP!7d(=yy&9{#Qt++yWhP6pY@z_&bO^i?j9Q{)rt}UBrqzsQl9|MhkvDVkI+8Bxe@);d%W^=$c}kQ}>53P5hBxv8nCsn6#*<^48q#($3Q zck%0W;>=t>$M-pmT>s~!il5)};UrC;_kC#NOn$ca?F;_=x&S}cGDJn=>);yZEIW$z z##fGtC(rvlJ45Z~mDn;b6=7y$Lwtk|2rm1)`xyWvQp7%%SYccxvl;h70$-ReGQnu%%B!B&k z@aiHq>{{Aek@+uBlfpW^PbSJ=mVuu%)SzUi-6cZFKOyoSj$5g~pY3er8t8C{7j;gd$K zi<=EY=)@ys=a}`RzKrw_DAb8Uks0P5qK)PnSCnzxKwhOMBJwudNXwj7p9i!hOOCC% z&rIl$>Jp)I&-xV+h>Rz(GvA884UvyERs?OOZGSdwDy_cLbgNP1QtP! zN&G_UsL01dd2DQHP4+cTp@5#(rnpE;R`G)e{O;3kBz$j6a_?DAy&hgk4f_dPe}hkv zTjqvp;F?__Wc3kNV2wmuF6ZD1oj)_nM0PiA4|}$rbU%2~u-$)z+AzTnsq(}*(JDXX~qDH0%)67$M@hY!=ehoG~wR5#d7GX~JilrU2bxf#a~FXZq{-(!1L)&+c|r z0yM{iMLR83thgtQCyP$F5U za~pX$w9C{p1K%zD-WmkUzt%=-e>Dp2(XRL%hiYofa)hBbZUcNNvq%os8&4sb! zB>cjV`DVuV<6G)?5+0N(XSm!oJA#@>-^}<4IgrZ+_n&eR$W#-djO8?Y(KRxOv$LSz z#qH-akndo#`F1p26C=-Vmn={ApC|)C5&uot<}#S{5|Qv^?-WqWDIN@7iC!xse$b4y z5yS}7dU?$JwNpvwg zDe^VxBn0EaI`d&PZ9aV(8hQ;&R?wmY4wzepBt?XBfA9=jZMzqD!+Epuj+Z5z?i>as zay&yEk?vpVREk;-8T7aK;)z{S$z`7riTJU2JjFy}F{@ybP^6!4m~0=k&lD|z3wL(*w7ekJ{NLI=LW-}89babdvJ9fbN`uU$QAZC42uyhd zogq&u-By0V{x@P6P8X5VT#|-XmA|Iv*fPw(Ep)X}Bmp022*UEQ0VY%)xk<~W-H>H4 z6%H+6An(Cu@R(y2JS)sazxIg>oRVykVsiW|ztd?NNPRjl zbeMBYe*U6LR4i8!+rBkzRQjgh!R$j*pO_gVxEPsaD=X-?yMzbj{073nBK88nLz{Np zH9g+e{{oF{VO>UxRx_-&ShBw7OJH~MgUvBzBP4~Cr0MBEG!clyGzt)i(H)?>EbgUr z+o$}xbd8myR7!B!J=2`C02cJqAup$&mly|v8F$cK*9Ff!G-LznxKx|m<48e@O$B<> zpGYki;OLlnDUuuB7o7)BR|b8eR=FQWZx4@y#k%hgH|{i(2Y^*??bcf=cE8){2;jw^ zjbOCLgYK{4{GL7DlX-)?yr>c;w|LOSP4)^=7$y`gzCn7!%VCS#X-~iv*V0}Ib0J83 zU1vfUw>!=SByTqz3RFI?IuwMx-gL=md(9HUb_AlmmfU2M@D3Qn!+%Sr;1yuVm)L2^ zN_s61hCfO@m#(s2FeG1Oqhwq3@s%8mMT&QBG1Pq?zWdWFp&*5o?!rj#KpOSnGkXn` zeqtxE6Epc47So5nhenYThWm^qf|TCJ?x2m8@zdw|Su{_&$I71J<6+3Z*=ZfT8EnUZ zfdtVrITJu8fr2UFD3J{|Z$^M+qTDn>p}^P622Vg_ypxH5O92W}1ZoVq!{{gw3D*?t zcRv_5!S6iHyMLX~E(Dt2np-mtW;%Xadc&_m+{Qb0^}G>XzXwqA=^}~r0Lo!}4D2HUc^{Y+rJp`Mn%@AU)FWg6STNQ{E zkFY$K=H)TKrr}qFeJ2lu_H^me6JFRn6_iQNYm8s zwX#(o!bIx7(8lZbh--_1{-Zkiuopo**qi@bE1;2)G4L*rh7B;h2kB#Rwqo7NY?HF{Z3CWLagev=Kib8% zI4tgAn0~xWB|Z$g{5FXZNH;pBl%w@K!zu6OMGvFu=S=STl3vB;Mg9dK zhXp`IiMMyRH;2URC(?72A%>UIr%hr+kn%H2Asf7`u7t{Vw6&+dS$K?wb`zM=d3!5p|2J2OL_J9ZMo)5V$m z&&$d{!!8OmtrI9c+LW!d+po(TfNQdC2FJuZk-u5~ zRF?%-F!YGJ#sa9hW}nJ{CDj#36XxFOF`i_IW~69jFTHA>lIk;#zz!tcd0-Nt zdDvnya9N{MUq0&y+1kQZn^j6*&7JpuuUJB!mRIiilRYdP4k0@6@e_y+9Ktl;xajyP zIWOX4r>hP`C(i#3l-LpsQ3HDwjTrdGB$lrlRiRJw&Pfcf~{kPhBI#>%5C7~I^ZQAd~3J^k)U(AN*Fz9NudF3)# zKb`6JHy)*UO~UY7UN;);q_5s~VJFIR2bunQQEyA*0r1bY77j+!P)c9b0w;|@LU6*w zgOn7CaiILem=P+D2nzdYN4&1O$WM2CG-END+9s79xJ*S`dGD2dl4c zhX3F#>}lJI_evi)24( zv4Y-pm)(|qBiBr2XRf>m?M9}w8uDjLb(}4Nva+uGSZ2%XW1+4n5XC}JW1z0wuu;`+ z6~teb?TQAq?C~+}Bb7BG6u7IZB};A!IA_LNZ?G0uqV>AmOH!C56NTchxHEqtUY5@9 zdD|%aYbbkaCxO?XtSTM3FJo=+xB!{d{t8Wg;%p#CdTpeJYi~x$x8aP@2{pzhS3{@v z(KdkZRbtUrthBW&LQ^rn`WBq%z3G2q9V(_1_1})&`{OD76S^tXKUxRT3`E97ULSgU zbXW)j9|7qii-?Y8o1?`UT%!s?IKqEFqRCWLD=xe*xQ^gVO$EYxXJi)KHHbzLGv-*4 zi=CTOxcc;cq*=x?NDd;e^nY5K4!wMRKjZ`4zdr(ae~+8+j@7Kcw}aZ$kR+la(i5ts zV&h7VA;A<3QGb^gHngwO-9t8MCl7PYG&Ki?)npg(dtLVqfDdS>g|WCPey8WzbqF5d z0-X#>Dsz=DB4_tzIoS{JKCp*Aq-c%ijlln{Dzgk7qD2POe+(Y7017YuzVQ3=r5oNq z24LqOGVmp}+nCcS4rTmFcr7;m)%R%7g zeoCrIvB+VEj!2ZNFcw>UiSBkzQ+eEW1m*d-!DQyC*TC=Z~vCMUF~5X6wV5R~?A zZphXaTtUZNoQ0Y9>CLDusb)BlDxnO>$VOhk9XwE9hUs3c;LO%E>@A70(HHb4U*wMX zMz&}df-&dnuHn_5ffegepLC~I01({CHLNmz?R3PyA2z|b?-aET1x|Y)eS2ga7hM`5*}Wucg!haMaysQ-oa0?bzoaNnR+Nvb=4h>SQs^q7Tlh__Xt5MSl zZ=6>Cxh%FRm}=I}HCO1jIv(-PU3aZN;%>C(>oC**JJ9UhezBt>@ZdF|QU4ma!C|*v z*r6}$7?8akKbQ=O|w2{k~x#k1=Iit#dlK&*eE}bqvi_{gjcu_R3lr=;QvU;58Uxy}Q0|CXF z&n$c8YX0K*00IA_t*wnok5A-THu}C51iCq>`K<#n+!NfI%mzfwu+tuw5{B(OI0ZkF(Xds7_Cv zw#U{%y({u97uqer&aRB`(oWC2)sOv_v1{wfU0uCK@TYoXbXiL z&iY;aN_z)Y)-tY8(cTYj(J|5b^3d zilMI2*k(EiwlC{@cC|e6Wyp}_sQ_h_L7XLbD+dRxXYYQa#`9m1=WQ#d@`=ttRlHqB zrg!tqu@Zye9wDbGsNlO(9GKpKojz?DePALcff*{(rka7xC=vZ@|9twe`8bBQmT@r7 zU~3Xfoy*jG8!pUKFxJTt{uN=y-@%ojen4^MH`yU<-O{o=DOAZKD>rUDpf_`}%STr4 zM84DZ<^1jaPD@)B3Vf(W;IMYvsqZ00uQYqk69`ooR-#rG4HZ3Vk++$2PT~i$`PFh; z?*OJs3T1`WCe{Vd1A6#7lh)-`JJi7fRpDtoc~RBxi-XY*?# zHRbNrjOb66uu>uulw#j_+*r+M?gJXSI%u%}(#vo_#1^h7{%`$9?6++#@h=n#mm+67~gXZ>*0{8go+ zAh9j?0pGgedfeRP0rGU4ghAvXzy5kS9HXzr?%Q=TOO7rmMPu{NGWP+g$d*FdL94_< zw=XxBRaJSN$i417wVR`t1TaWp$XY%JT-a8``@n^A%pa5jB@zl*{Z%umIQ_6DLZO)t zT<}k6mUfArc;707oQ^Qe+PNP-Rb3)Sx&Z>KT5x7dg*(3tq$2M@>eWJxuAHHSAmXA% zL4RN&wNA{D>P$CCps65=paSI_;mi?O%aD|5k>Yptmc?^B@6mq!i|{uWb)IM-Q_woK z(5evE;Nx=SS+-^7*{IokZ#$BzOQ!|YYmg#qNVA}^(;}tffLg?DOG9s3O+`|DP)I2C zt&++-;c6&DZUIbq?k>_~uAU^W_x~ZhmV}of?KaagjHjBkfR9ZJHSjm52TQS34_VKQ zsK(yvy0*SzEhT$tZwR-105IA zT6?S?Z1^smlh$&Z7?eRHMT|0GciEIx8MOsi<*X?=X2?{nJD6p|{)j?r7<{co7J$h8 z40Ct6Hu+PoormJgWGMnhV(SAGX1Yfjs=07-!;8w$;<7UhtdFyhbnF^uIb5Qu*NDgz ziAV{kG_y@3Y<$_Y2nq&c8*40FA&{Mo%Yc(F#|&iZFp~>ATzWt>S!LU%(6`}|`OG>~ zk+5XpcCi|o1*s$(|7uUKz8jqq-CJg9y`cu@_ye)n6FUiCy5MQL*(&3*U=lN4)axd$Bk6&$&-F);n9>JOXo_^c4f;O5x;qH0fC)fejjPZ+q$X+ZZSDkJT}CO|C|-(p6%D1F9eutW$ZmGrZ*0 z_iA3Kz#oClD{R69Er*(?vl~cVniL*E7pYRQ>Z4+Qe8NgsQ_*l>+aj3NDnL8e;K8`% z5?-U)EiYx9w>P~jT{^u$R=J6nPasn7By@?oXq-pu!l9T!6(Gw~_F53X=@+Fod!M@3*Qsiz9Q(UA$Y&wjqgxO3AGl>t0rRZ^!uy-`{E>b9TR zq)>%W0CMFGa2v}e8{Dh8YYir2-pxSAG%1!`M!l(RJEtC zpa?=U`#%%N+o+Jc7B3X~MVYOG`jOPTaxEJzf+eFwkPdS2oPU2XL+1Mx1BsqskJsN2 zTnMrN#>)X>xEB8li#n1E%%RExB3}Ww#0^%f55Z<+X0*g4Np^sEmq zAbTM9c<2nV#&k_wzR|Y(&X)Qj{h^k7xm)|AoZY2S?r)DW`)BX&r8xJGd%3-ZpPk#M zB7NiKfJwGXfn2srfdmjNSAwnobMoV_$6*B^J?MR6GVX)Oyc=w8e`y{A3}NvV{@Khf z)Q`XD1+s)6UxfS09&p$jvShdu|7T z;Tv0Yo_RzUU>aK-8d;>SmoW~&xeauY6&iCFmyRm341MDNSrg=5J>?Z!J_lH_qXz#~ z-x`XhYpJzL)MFbvPQf`9gV7zclv0vVv}+|2ceYZ#W)(j+8(@EJK%|?gg>%wc>9L0W zyBF&YIr-?siqE|ZX4mlDf(s1d*5UKpMPL;4mKlEOrd~P{PAw0`6fK2pQ40@6af%=^ z5=q0&=#Sf&pyMnKdz$PhOFUW3u$*5X6$J~3ALrD%2gto`XKZq2mXf2NxM7#?iI+v- zxdjb7wpJ0c?}Vc)fylxM)96}|TccxH-3u*LRDvgn?!Gi@C}@Q}E^x<$#K_SDs` zE_%b0yziUrZ`qQ6B5#iT4RPCB8?sfjsON=(ZImoHaHC||W(#q{@MH*hcf(Fx?5+z6 zT6MA5eDt{3m5IZX2*#6Ti)ZLzX7ysZZ%4Vb0BXX>R^iQjn7<_3vL(N=c{=La(p5Y< z_mO8C>e;Tts^f;matrHS9^DhXdAb5{{Tuyjw1Ww?hZstsjKhSukQ{DM^If??bZzzw zr47t=^i?~)V@Iist(1DFdOblGM)`c&vgKuj#JEZF@{vT~e3(qB{O8s5yU{nQUZgEw znb`$&IL^N6BJvUOUq&2yzeT=UO$)(9>_%+SdJ~Z;tPXnuS(?g|3!Y)=xuYdw2GP2< zgQJwEs%6?JXY(d5r8G1Ldq#s;R#$Be7nQxUk!BSd`dC9zxfh!0@w)fHx;FT=p^SIb_IO*@R@ihKaio!bTl~?mhG>>(=22?p zFust6%1pl2sO3QDzrC@0{;M4t4cao-n!6TiFI+@>jTS3>E%}y-KxwHrApppoO}0t^ zL&Id|^rG9ohA3+k?-zAoh6p&Ez9R)S75EZW!ahO!*~0I8-&hHCrKW_Bi&)HT`2r@Y zT@f?X=6N($j$#qBm5W#h8&{E-`SN+RRqFy~OWzp&X>?^RXAu3wGj^G%bylZ_#sbL? zn|hx@R^Au;X*;&lqZ)2Iqj*B~UMTkZ(L=9Jm;2aG%$8f^1s>hl395Hg=kM;!@T=4Kdh&Rgs~zZ#p9;E1aZ~v!mf*)7kXKo58L9N%abqA1oG9 zK$1F3FPvHZJIZ!a0Nnf7Xy%l_4j@r>4wl`h1^%r6oGEnQ#POHkplpoFJn4@!8ZC|6 zxQTn;tBf1%(~o%SI33nDpIH;L?X;P>HglHli2-zR%cDXy((XuUi?IJK-Nsoa@I6rO zI*~HVT%`idQqQyo#3@@W{mbzcp3xC!4L-j1UqwX1fS|2*_H0u!*PO6IW~y`oq0YYW ziZfHhy-8vEg#@(bjnR)<+}Fp(T-yuVX?N>G7DeUR29yKwhxIxajT8-zTLQ1q6dcDOACCVaPTVkh!LY(kDSj?Ks4yXO4TVu3@X5 zHJ0#CrwCYwu#qibH+jV6+g&VYP{29!wX1^Y32@D3P`2_dziRv#;q#v}n82LoEb9wj za-hv5Zp+`=VhHxc?@c(mMFFb(0gzjF{1mu?KPz*uSa?QeH99D(28ws=77JdCUlUx6Ezx9|nV1-NJV+nvMJc4O|6!*{LfJN~^ zg~PDr;*ayg3n(}-aA{Stb_fXq?n2t3t?}^Rk&EF8 z9(%;UVT5DT%I^qvd<#a2Q49VYvgqM@07>~cn9LQ4JoA`|%#Fho5K9xRilGZq z!`A)#%SIRUrV$t5h!<0(vE5=gr-=xRiP&kxURW!m+kl(f9z!s5Q>9K}Ml9jhmL9t| zo6kTHdgOS>{&&=FxPVWQ{9y^b%}^i}v2<)bJ%($Zi3hU4BZh66>2O?R*ve+a5Oji% zMUL1Bd@SV8@QB$p*dD&(b|_`1cR}`e9(vo}kDT}CQfz5mDpA|RxD>Vv1cWerI4>foj!g#``Bv9-}-V_aJqJ~lFZ z?4-DuiLt}Ybf4<7n2E8`W8;R~=!r2Renv8U=w` z|4EfAv@{^ij*ZujNpbX%$MUKTV!pd|#iTflN^uyIW-nl}auvy7|IcJ5`3L{Y#LQLd z|Fa;~tNs7{hY=v`CiZ`4G$76Xw?$^P!F!%Z?&{#B@aGHKup?{}Sp_(Xu%m)=AI1VF zjvn3D`Q6HU=6tf)FqIi1C_AA`w^nE)S2X?adD_i5hu5&iB_9$!Pz^?uy;@cx0s;rbtDl_8#rge(Ea=0 zrFq*rXM*72>x60D{IB!-f2*$!SNu;{1KGSfLb)ZbiY)dlj)Pc;7oF#J$vd+(@N`K_ zZ}y_FUX^P4hJj~`rh?NO7dIC%87lWIbZM7{@#q2Sa$F!x%2*dr>i`*)2Rfa?9X>j*nDA>Fp+SxGho=rnR=@gDq5O58rApx`}wD{~cGm>Inr2#+q? zu{EBhh1I>4d%hyW4IYk1`2YMgUewRRXV%XX@?4+Wx+0YnXyiqnLS7yIJ9}2oR5mxJ zNcFIVd(XDUt;Rc8$`FKWE_lx-Loq*Bx)!9~TO+SC+U@Vl%4W!}x zM)_G{KF(sF+v5>Ev$T>fK~0>FnpPvsB+6dGkC=C#L3R<>^a_N^^y}#fM|>P~cHW9h z&$8yg!LDf%EPF!H%~D;k%1x|apZzguVUf=>%Qi0_ve~T*b=lJI)*6i8BFb(>HS-V* zW;rA4Z;|O&$)a^G@E?*WwE0rcxkD*nr@2~mzr$LBnQLq`!5W`<-guhWc!HUwQ(0I< zxiq?MT>w;ENT>Z>pnM*6=F5HDd*+0{Y7Bcx{&jd>98A%$LY-GyR}1^ZrmHquWF*}R zdmrxptsHQ*TJu)8rbvq1cZr<0uH|ZTpPAT|(u+Fpenf6A1-Rs}z(-MvxIu55c$43J z&KlaLiGV(jnV%$`s?i{t{x;LJGg+-cxMfwT@svEmuA^jwxe2EKL=?BA)fkfwRO^uJ zKVkq=K&-#^-5z$A$`PAEpj%0%7e*kWcI@A>jbdx`(Fr|~so=<_rqg!Tih609nMAB5 zLnDJ;0`5>&PRUzR47=cHL3=VKAr_ih$=D_6bS1?jxUGAw&>+0=Ahrgujh!Yj1+62! zab}%Wjl-1Eim6eco5cMP;m{`R<;h2tl_rQ7wn~RNaIz@Ax4P|nhV5K^K(vQh4hMhVw*1h#gxW>}vTUf=Mr!@?;#58ykw zlQaGu*J8pCpx2*wa=N*^-RaOk9TnBfL@|)E#n<$&>so!fg`gkLgWP$(Tz2TW4tbxM zQC5%8qWj1yC(13vX(-IER)54JMDTHc+<#pHt{&CtKV<8@){@*g%_t=m39FKcTD`-< zy!vV?_HDqVCPr`!n_Km-NVNw*z!G@0lb(uew@HeDRZc@1v)AK_m$PFq;gaPRob}EZ z{9Z7l1>&0}Xx5y)PnwX;pFsa^OQTDDPlBG=UAI^YI}M@uy)L7G@x4{Ql!5r$I}f;~ zO)sz%P35oJg~DB`l%9wDut{1&uSuiK(hyq{^_SVD0YD)e`-x`4lSBWgWGXonNL)Oc z>&PxNIqk^MMYE_ut7_~{C=(x(QL1?;aaWaz(PokOv<%YZ7PBhx#GM_U=FtCK#^tRGFhW&_oj>3*qsKwjtB1ZN9y#X=4Y)f=+0)IMuo>}3i z+tcapg>(wiTl_(;T)t_=&M5%iv4sO2CJci^ULLY91?8vIKKnq(vv#iC4ogT)ZXKSp_mBTo838aQn$=74>`(Y3;iN3Mg z-4|-H>veY&vOkT4BTd4^pcG8^lN;iv(4+e@sW)=O>7pAvuk1aUq*1A7{nI)@+L4V?r}42@+Rmi0;6 zMjRTh^9*7Pv2Yk@kZ`y^=PZwDq2C;Lw*opA8mBI9bQbe*L>HO{Imh=-5n4}$V{;U}Eny39H7xvDV~Bib7n z`pag$Fq`VsMGSrS(l3J?7Qe{zf92+F-=&e)K9>#-{C$28@qax%-llA)@#=QL>Flc~ zVTtw+)0uLSX7G$G2BFpBxa5)a7{Ka)x7*^ENYgG48^6qFm!hU9I$O?61)2*7V0sP_ zxN#&u&HjI6`crVG{}=QNXJSpviEZ1q?POw`cWmp9ZQHhO+vdc6>-qhkw|4E?N7q`b zKiyZ=_qfk{A=`$K4X;X+B5okmV1*}a8oo-;Xhj&~siyVJ z-O~am|G-aPy-#@I6`yvf{|cXBV3^yOx*~yhht|lDs6FsHl%ofpav+{4#U|)<|5HY5 zgd-#@kfn}f?kHe)ke_7$N*9nkOj;&jGL3HWcN5dlvs2vUk{To=_rS^)9e^^7NxcPV z4jey(x)dgEJ{l{(UBab46`10Tu1j^e@2J*{UE;BIn*YY!U39t~gfN_LG(y*b%UjT$ z5(8F0g$VOU#wIPWCD${RrrGX@;)0rhtO4J)TX(UMWLNW>xTdHXl6sahlOCV9#N-74 zsm?GO2B~AMR5&@fqR12V%A$3pXMO}TQLxXLB zZx?TGm&e7I+Me!q?|V4D? zIhfe$d^-ugan74sn#CE;{MoQ>e~WIZ_nxfYI_Cf>L=*6X-~%&`#NqSv$n5+B&9m4Z z_E8y2z~3lRAt8axibxzLCaJE%D(Wfx=f&C=wx(%s>cym}Y!t~j{EzzkoxTOllq8RZ zFnaFJTwquSa4qajOww?NJb1&b9RA9SLo5ay8uMzPB+; zL6UO9^OPIO-e`1eR1R*(B~U2x1VqbxyZHjgVo+PbL~Xm$@c!l^IyP|6bJqW1P%a$a zttnu@G?C}iBx!<3FiqwQC<&(Wk$_xtqnDJ|ys#30FzYXowUd2nf=yi)_%mR!KbSt~ zx~lj02)_B5=Mdga1b(?#AWftfiVkhAz|msaTrQl-=Ba zrKi3ET47 z)Q>hjMabDJ0%LHDePyt=^Bu8fdYLL9KjFhXZ;(=&tL_(Q@OtD_bs@3?G*Lt;!@QMT zkR}P-?-D#h?STtC2-O6F?PGOkdzoJ4HPg&vqZkn@7IP=gqIy*ZHw&q@MWvG3^uQ5``8S^ z4M|pos502*%m-K4fO*5;3#Z5PR@vYpc;3fSM}K5s*KO4ZSpRp56g{Nf#IzFrb-oh+IU_SGK5 zNWP08S^mirs1lCb;`lNDLFlx()Wej;gsFUwuDwCeF7o$Ed-Uz(F~_mZOUTKYm2P`< z>bj=aTHHXqL8cme84BYZn)!IzHWq?~up5Hd$9~YccJzZnDfMeOdV1%4>Kd6Ip!B3z z!5(ynFk=zO^3IS@Kzuqiz%AHDs+j=OCJg##^ZCq#ZbHN!e3_ve@qs~H1J=dbREIM+ot@Ay-8d}8ik4tGjG0mitZ=Ubg`QL3`IyT>IR%`}7NU zQkA?;=(*T^B0AnXeg#myYtbpIk}5&h(I9EcSSL<&29p8g!UUg&rd6`sSu1^}mKxCF z)`jV0zir=ibhV)C*Ze|<{L=`4*F;^L&+U!gv2mQD@oin7zqTGMC+$5M-|nlNaqiwm zF~;cxxLH!3C3}3GlmrSvokCk@KNL42`j}T+zmt1PIH^H0%s4o?$BDtJ(^Q>Bn<=r~ z+Yut8Vt_N*)j6Li-T^A})J-R&+pO<-Ds!iqru%rc857x|`-r9VGhjr{`!BBNZ@t3o zvnDHEE-(QxWovfV7noqqAK8|uDMZ*~UuANSO}oM~;(RG^PStH1Sgl)#Cn5N!LCHm6 zU5LqTVwOTS<4F&bQ41MEm-+!93341-q%!6&#-bA{cPwb_?X_S6G~itGBos7Q%Tl-- zx`+{s51>jBOgk%Wgh^6$DhvM~jHnRpl@1v*dkB@i9mlXPZ1It}c8Mn+WcEi@19*%ESU@lWyb zyq8t5zn6&|vwBo&-uQ3UDa83dhl-v5#CQKH>HjTi5;*@CjNz{^`#*11 z7;j7j(K>BWbU9=_w1DRNjmFzY@JjN12;Qb4epOEkC;mmd>5BxN!S=0A=4Kj zYa*6{4tUqT`Nk%GA8xubOyB&9urIcHiDuwOoUxM3)(Yp%3G?-GeXAM~VREKTYG9lF z3t0z{ggf+MT*6|jwOn0*Rb3Qr^w-bK8k6UOwfjeZTwa3(0X;;I^2Y_aDF};|2x_&Z zBD52FwMkCHf#iTX%e<|kACl!Ak1^R?CSglY>t`8ATBM4Mp-LCE&bnWz(mg))GtJD; zqi0wiKBpaYtd%ILl5fMLS3GL?>mLh?QokBerS=}z8i$R4ZzUr(Movt$oTta4BK52! zJyu~ar0XgrB=&xyH?E(_l~!zfD#Ejn_&L-4_h`p1gZOdeJ)ijG@%J|+G^O}~8*1H-CZ z#u14&gP*^Ts-tBdO6pyP7+pduo&COoX#T+kZpf^)%Xa<;5%^~HUh&QJGq)T|r{yyTSXGDQH2o3v zXHlS5@rx0&s8=s8&e+4;@N4WY3BN{gKFP$EC+=#}`NYBAZjqbJB}Z-+mvC=&I|w|k z^35o$aRlEKym7=-RF!2U(@DAq&r~9{=`re=Li-1k4Dk1=gQD$_($Dybe1PeVN7a5Omsk<5`%Qf>SoelBWNW^x zk3X4_nuD8!)E|Amv;aOJP%sYwG{tfLu>r45PlNXu@*|w!EPzBSU+I)~9f#mR8IYM@ zd97Kd`=us^Qi5bB$}H6}r&7S>IcBFiXtTrvnslm5>XKqRhhlEJ;}Lx$`~rA*qv{}^ zNMneBp5@f(^RIuy0;DF2F3K?nY5~JO%8!>rF$bQUwf{&~de*b@M9^LyZ2Kqo3tHX3 zj~w4V4vJX$I8tet(n0yrzh;z-ykDdsNn?9P2-kciU^32-N7FKvVWdc9@LBd4ZW+>s zTzwGnrJcq=FR-zpLp;HD%DD`29)OrqW2t4vBkR$_g(&;Z##TRtNe9nIQ|f~K(IO{7 zHuOjd!Q%J}XUB~RHw~j6LTC|yj$PH9*8XA-Mm?D^1d%vY2{I&nNUVm}LENS2)k%^L zyM-<&y!dj9sNV!qqCh(?fBd=oU4WKk#lLB35A*%!sHsXCCcN*owsyCN_oo(<;iGj6 zE{pc^%@BK_PsLw@_CA9!8dm(Lv+ldDFrj4h3oYB2qYNN~yde}~u=hKPZ7)Ia>)o-ovRKRu+>H>$mz@6-1EjsirkY{sQuxfMdA$G8obRcAOp}gCE`vp zWOc-F8F}c)aDh%$31r&)zopxd#BP+9AIItjR33k0S{%=`^J{Bwx81mfsY7*rlO#6~ z{(Br>HxT%lD1;J_MovRfg}izAqmA1MV7shDObFAGoWRiXQFzu-H-hU|-TtF^m4EPf zvtbnXyH>451e3@;Z976;Kh+&U2adir9+Kh={tL=8>>G@p$?w^k?tadH?IVZkg7>fQ zca1{3FPxrkpjgPC5GoBr0u7w98je9h64*fh-{}w z6PZCs(?ec9zko|Nu#lxvt+(WO-mUuO@r0bpf3P5-`yMVs;CN0#?=w^!*Y=2<`klX@ zA+GD`hxnhgYxSSr|698;>_MGJs>Cy2qeX|gcQR%qYg$M;RiU^dwlfGpth)?n=3>aQ zv4WgfyA)T7RP~=tkv==A(DL2U6_*czpi?5jBBxKVwz8{@2xrd^Hc)5nk{dRW*P*`D zCU$tMK7VY9kq`rFA^3e-1J7kbtiFfegESd!$NaO*Il)n&?C3-ajcWB^eG<-4hdAVAcI~HX?9Dz&V_m$qNgIJfsM)oA5S^Z~9BzGU( z7m9i|Lp5*`th&w)9s4ytUzp14Nos=r|^(=peV^keA&tJSTdd-1kI1&<>vQ{?S8 zHL`(H@vXdeav2OtFIBbcYMnWpFmD~RSoUN^@iO|b{j7sHuTDT_%|G}^;z8_mP76}q z`;uemE>hG&^+b0VqVnsbFfUbOTC9B;3on*s{rZd+7Y@g^bZ!sogRfsHNyRR5i=gID zf5kAvIjXH2X}W$LFH@IwlRnuMMz0hSC}c@(A=OniGaKP$nZJ0D@ZFp3I{d;|!!R5Q zVJTQ`zBzen{~_LB(~_&QtCnrE(0%oa=m4tML3h9ZP13ljnAhdemGgI3e#)G_$1&y# z7QPUeY0uI8LT1iQ6y ziN8)s3bs}Tl`f1B4OV?bp@}8$J|e{22NRg0xe`yfUa}F^Ek2gFLL#LMUBN!fsdA)Oi`qN>RGZ1b_S!=d z?qO#%@6A6}E$z<&itP9f9zaYK!7Kx+i(WHQ9rv=}M$dU@`*|B!<0zh1q0|8`p^&{M zGKoP&drKW5PtK)4^UL?51kzV*ryMGMLeN_ewxa}FD;|@%AKL=yzNX*-AGVG<&|Ddg zt(x@>-SspUqH_UM#Y{Q%D7KE3-=MnpSn~|s;ai76Z_k%yh6QHKdGd&e6Rhy4oqN47PzCtZ;y`PQm=K<9T`x*)DVG3BwML2()c^UO%K9&mH zPWoVx9Fh2Qmd$0MUu$y{=x9cOPpT)K;_>%Jtv>^Mi~5hnunMzSsuE$ltaWkuX z9eRp8DO+zHX4k@R=v433;mo;vXM6!gFL-_2&+vEsOQ00f#QdO}%-n3;A_wiJu47FP z)q|zzZB{Q%Ow(~F1RD(4HO!jZWBxw{gXD?AXu&y^WxuVRP4LjzOv<(PPxZS2nt&Fx zDtm^UOP>NZ#Wo}+GWK{ba~d(Tu~sXzV}eH*%j^hPiZK2C-vjAKOx)$J4=uGu&)=m# zyoXlUKKCLk-_3FlMWG{AeN0KeiL3icGvE^9lj=yzz*2MuYT*%f=|3QF2Usz7jQO4bce?Q@%|Uw#;gGxG(pPU;%EqJXn6qU$3C ztF?=uO9YP#uZ2`>rrBWnwRzl-0YYP42;vOwh4|ctV|+?{Rx^42^;ZQtXd+>T-e(bx z0BI9UyZyUZ;5X_jkk%?(Z-{48QcX5DSoUl}Vtq0nSJpQWTR(+Rn8LNb>&(rxG|nIK z?VSi3wKPQc(P)08nBZGz9|iul2WiaXvtR1iF-$XNUK_K2#JfwjtsxiATnpni-Qs3H_T#m zptP088i8mfd$$+QbZ%|rEb9>Q9Y3N9ZgRVl)>LN~3qyJm#!^B_l1wT!pz&6`_g=vz z3Av_01C=q3!-V7w-_L$c||phzApE00cg3-b>@hurda|L(Hn^ST+( zmt_WjokGC@5->U#6|n)g;oA;7!;-`Dss}HOhaH0*bUuI$kF6jz?P}%F%RxRSBY;_S zb^zPV>a0K6B&6qHKoV}ut2Fls?NsUC>M$}qcFh6TgE;k_l>fNFS|7P z==<#Zi*wQna2kTSCIaw|6n z*S2`o$JJ-Y8_$l1eI}%D7g=6Ibpq*1xYdhU$H_M=wyW5YJc0hGeESNOFVMlrW}*RQ z37kW(WMU5Y%tSdB6k}j19@;rdPAZ@4cZprF{zbP`#&t35=n7=&Wpj64%fBl~s2whI z@z!kvC#6CQGEoE>`ggr2eCde(aFjmmeY$AvXkG_(dlzUbFWU3Lj({WH$;T-}kmt}_ zW;O$7H9(QOyMl?jTW`8|uueGoI~9B2}F_m67!#{_f@6xYD!0 zm}i1|lB3W@xf z+n>Uq!%b@6Rm=F5IM$d2CQM+-Jhw?&Z2p=lsJA}^5?>8ElB*WzZe{?{t0W#LF3d%> zNX&VQ@ohq>4)oEE&>C-?DRn@G_GnwT>M{!b#yf!}$eR5%0p#P>l~gi(*~BWTg6=5a zdu=71%0cZ9%vB+HdIvlpl}yWsMAyQg%7n#VXf`}%Z*nx~mv zeLa9(ef`|ZI63f1lEy^!ka|xHE}NUeBtA#_j)w zRI{tE-#5i6bYoXux%`he|IdtFc@_O%7~ZUhqa4-sv?FFB^-$nXWw-__&583Y{yaQ# zPX~m0S%ZJGkECd}AbY}hGnAkgq83g9T>B9?FkUk#AJe{v%3bLFhx@|#3zotOlgCM3~Rpz@Um zkP~?e%ux?Ly#Vm$NDAY>6cbHLfe=rLUiCQiIoxXE z=8@I>5Sb%baj90r4vB59VhCHVn4ok}Kbwi4q3e0~#r(n4NRr|mTR@gcaY8W@8DNdK%+b`<0~d)q|cThgJVrn0hGT2PZ-Xb}n?= z$iZh#Y$#ok!Qg}dn|C%ziL~kZrct$g9ZrKZ5`nehNghJ7ON&=p!#P2NvU3jJU9%CQZ#} zf%3wEC{U35z_On}QAPNFZ(4`ogoRnI{i0T?1Fw{h3ch3GmVg6M)`Iw$Z0!BQo@UqX zcvfDLE zy;GD}>x%I7;JX&EbJjPg(Izyp6X!dD;Rxqh7@9uAXSVL-Vlr_1P5D@3d*XG#(-u#) z-C9^~svezKpMMLN6(CW0Nhu=wvZ@-Me6Kotw;&2Q%G`;uJjz2^Fk27`TLxQt6CPhM>%R?5}iZB zneRopiv?z7866LpC^Jk#B`RO4`jpT#q>RSEGBa)PdUY_WP*S%(0*t|(tck!#JA)9K zT5P5F9F_aPAXG~=ZsZpY@81<_-NlQ18MBp~q_6Z+AvOah8kar?L+jzWU%n~`L-qj>_O|0Vq55mdJaAyYacte9b~-^~4$!J@pP46kUbz6mZu zYkD^RIKFP*ubt09xANX$Qhl;@Rct_?dh6fQ&5YBUQjywWl6yzv14F7-Uenh_5FlOD zJfVGc?x+EI9ljq5=4Hy{wRg@jx@B{6aktuL_qXr)3qf3w;9vhQZe|7ijK7s2Nc>Li z*LMb8lVC92+Wk&kEAD{8{?`-YvJuu}l^@+tOsxa900+hlL*(dUWn8_?`h2Z?BtvUz zP95YDq`trA*yXFGUsq?N4>WhkVFx|37L|B@M0)$x@03;qQQ&ELIb?3ebC+Pe^D$B{ zMrxF0#prpmJ7ii!=bi+gMB-u8WEb{j!s}@xr%mn}?wAV#Nt^rec&3XVojaz?0!gMr zQrlcs43p$Nt*He`q6W?{Af_E~n4P0(C(6e1SlvAU4dhL03*8Tu!eE`%xv{w8W;(Gaa=u0m$ zF{fXZ6MRWtCvI@26To<`ikm~0f`bnJZ>e#tlGxdrLJtV}kso^vk}!8QGvlLmJwV0H zLB$AE$?rgv9F^#LllQc@l;cE#KbweY=l2|wyNhJ0XyX)O*PZ+7Oq@<@g4K1I)yA}j z-CoA%S%5y?(NC9GYDpv4MWQc-xou=*1=C;yxYJ+HseLgk4I{2ZuDw^Kw`TjaUjstw z3R-nXOy=qWmABk)&!COZa#4JGnO)g!)pX)I%Z5RO0Q0M9P-*TmR*o}GtF|{fzb*g1 z!}hFD_;2NEiy0Oy??#wHmzuV|(|puWdcoWjl%9>(6uKwkv%S)M+^x*HZ^QrqC1Sv* zn)e|H#Lel=AKjV7JC>gCZkU9KzV>O?jhCe$bOO!gncj}!wL+^$c{t@PJy_<_TTswT zNWy>PcHWXEOd9!36 zMdhaA$*B_)1@Q3;%o!MvFrsQafxi1$dv^NsU4PGCPDj+D;hK*WQ#Ra19=bYR`;U=mbmOGK6zDME3V7=RID;PXu#LPH z!+v6IWEI_43iZ^k9itP&wmSyCAui8L?{edhpQj%<6rUDONNGsZG1K|zRDzw~Nstqq z+(6$_6cmLPYrlTs0Ut*2OelPBYJ{4%+8Vk}mQr z$FblITrtsm#(%-xUet9#W&j=>pPSABAnV|Mi#m(>EHMG|&4RrStIHE^N-V9jnjZZpH+Bt_@1-x`d$7+ z2;XFIH~@As5bxOEuUUkdaO&r)t2g%+3h=GyYd>}VMJBVM#!b{Vs(C15LNEAAFr&n4 zylB8ROPQh<*7MsnHSFnB#{IX$d3EaH9O5pQr^Pr+=FgB=CMVX^FxjjARmLBKN3sH$IW5jxXKUw#?8qSnJCLh zD-=K`J>(=SkXbq-@mlk(QOdvTNJ-qH5UzBWH{|{IsPcL%`r=~bxiSB1`z0A;g-R*S z3@@w-$Bsz|YLjY&;tZED1kcYyu4AhN)rHA#dbyYh{-()H+&}d79~VuCE4JL4=p6R4 z6gMPgXMGX6WSN=y9E**M^jfUhJDrI?ZP|?gaEvVVR)16&g-HvImCWsppDnK@W+ilw zBnoam2|QJ%o>TiIaG)rC!1b#k0P{)t>*M@ZJOYiCB-{Ur|7lbj1e^EXE_-+O`T@~m8z^Y`@x zCZii|7fTSbU8FU#Y)4&8typOv_>*BLG?B0gwnz-B!zd^@gIwM2$b&SpndMZpZe

    z;{M)l!~{fM)}1)4(4a9*?(192)kuNC=L zc%B8LIo+Q6vkmIAYRCVM<>upfMz|pgvo;LnF`XF`(rTX)sw+lehyg8b=vosyTu=TD zU5Y8lA8!89X9{Ca<>*kE0LJP$8VwKzaZ&SnDSmNt4n1RA3fzBMy8%;#* zhOoU}k!MM=^Zn+{0(h@pnQ%3oCX4F*XUrAhec0S(D(MB7@0&S;g2Qea*@q)kZ;L zm`_2Vd7Ou$%{$U+8~&o;N$P&Iw+<5ajScZD>R&WYq(4aYgmKE&=#uB9h70#PnCJcb z$pt~GJ-z-ec)Jk8IY?^wKI9iqd8$lsyAZ=X|9>6h7cqg<@B_%N?(2P&`>njxNA^^# zk>BqWg(GB;jzRj7y#n+?Yuu3{ZSjpYgvV~(CE-JuaS45g5IR5M5GX@|(N$87Bo*AZ zOzcZNmF?S&KBjkAXBM@)I-TzVF%D2f4lsOPUiY8Fmzh!a^!mEj5X?wQ_hnRCp?BAP z4%9U{a0ShPblZsn_idcTM4fqr&UBoRVuxNwH3*)F&;}=XG{Wa4@bDXr=(YM=N?A}7 zkx26}Nf^awbph)@GC`+26sutw|1wnm=_0HEVR#XQ9ge`4(C+jV7xs)aZu88Wjp;D< zQhx24N4Vt{qgB6;SDxVD|Zc&`~C*}AoT?V zkzqC;Zuo_Z?O)ptyvi?#m=Yz@L*;@eeiXzse>x>Rl%>mYEP^8YbNDZt`7IZ^G zXf0FCYoB5zzstq2#D@^5nc8qrmf6<$2H)lkhr?wfYZ~C?-j~){jGvz(DdC0E-wCr= ze_;5NP+fGsQBc^&auE=J4nHt|US|gf4eX4cJ)Za-0ExVXLPJR^e|gC=_=e_!F{|)h zb4a`X!s<6$+Lp@DFAtl(&1RKirYgBu&$^2Rwrnzn>N|$x#TNNA20M|-G%$;C^vvV* zW!jgmmsKq&vy9bNvL%`=?*%=f&K7s>Kr$D%UtS@ofFKF+_%>eBkm@AmTxzG!fMy%$ z=QXFVOJLm~)v(6t55J6L>gxV*Af6}{BnRYop$%VwN1-{cEi3@*>kUDvx}y{&-toU^2Ej2 zDb?rQ=wWS~pX<--;{E;nyuxnp`}4`aVTQ-o)m+W`4wj$yr!$Iw2dR?8qoW4)6yq0G zh}6f{+LUl|jcl`%$%((wooS2LNo<~o3dRy=iz}gSg;20xetPV#Fk}@a3dHpRA5MF$ zn;%^#0=%G8&3MH9x;QaV{m(oc_^bs;#G~xm2>9>mY0zDxGomtU8|Iofk6_$tz4|>R zi#iLpj&?k7c!NpB3Z;Q~Sm!vfQI`LoZm{0k;KnIDEln9aY|t;sxQ38v4vYWW3%0o| z*1oW2& z;V~=wHVt__>a=@dYmtq?#y~GQ`_HK&8Em)y_+bKJqxa(g&3^$ES=FkA)Xy}^R{E3g zP4WQzF$^ZGW`iVri_?)uaA_oJ{nzuGgTH|ot9;c+)RM)0YvtLbm$EQ>*fMImkjg+( z4+&f*Jvqe`VW%UbTCZ8ZdX*}@Kv=Q|JZtd$Kp0EdWld+_7`D~5`v~?`C{~4xA7j|} zkMXTf+hI$+cYkeQVNPR0N9Bau0$=TD?e3$}ro8rMeRXq|iTR-~9$G}jKsZDljIxty z_pT4(4TNC480cRq>8y04nFtxRBeHM(t=`@b5alz;1_FKKP+8e$FszmGNmm9fyC&yt zW}aA;Wt1S(vC!HlUW}_0ba4|x9qO|`bz7sOvK<+44Nh#wANP%favP7oU+WXZ`o)3a zFjTLpWL zt~V0s27yBx08h2Sl-SDaA#1}z=_?%q^SgELbG?ES;uvanLC8|aIIO7S`b68(7 z;~$GQw7su8NHtBSF&bbl&~@V@%avh=JJ{-Utrf$RQB{MIP^*(FdR8LU!-w58Hj{n8 zCj=J@{&MYa(oHrSsa?qe?K3oo6&LHs*y!c>InV+9)MfQnORcd+7M1zpYaH*{vW}L* zjk4h0NRHjlNMUO)l2h2Y9m&!o$~sKCKJbc**sjZ)K4}*jhih zgT=x%Fb9slJ}PUqwvA~r5@MJ^np4zhv>;%!4%k9^ZHgjWTzZ?8mo>jJ6kLBT=a`J9 zXH7v$rzcKULcAfa3$np?(7Msrs?}#?ovmgM8UK?DyXg3&Ddeg|vyl=nd}175GCk}z z<~FudrE4`Y&RCNRU~9X;T9mX9OwT=$2TK<_NbO~M@eW?srIt=oTLC? zlxArmRV1RtbA_)-Ho_<9#B=pOi!L=oot72D(^x-CBP?a$m#}e3S`)t15Erb|Yb`_C zYPj;mTQ;oZ_&xkiXcFi5QdO*%*U_7Gf%|?!^n3k>2xn7LEDF)y<>7oEcX_KX=%hMP zwZqc_JCK>!CbIet6ZbYJC4R#=VQ&SWmEzu!uI{NNaOWv+O6|-XDBtk21AdYRI|BOp z za{a*$&&@5aI+?Yr`7vh@<0@xez)@AD_wOLVjKjTYqTfo)y5|TYt{4|3xTvfyt1up4 zotfFqd{rVb(o!U_hRiwGG>hv^-2^+kSHJ9WA2BDGAs$!1mM zOL<`HJ5vfZl*n-GODO}Z-8aIS{k)~YXtfz0e}wM^jGytqn~~xi{8q$kE1@f(TN$K@Y-bRr#9iOO|cEMq`tO^EMgz-7j@>i>{}( zy!d9Egd9I&+13K9L6G$jpQC2s0S#sWKHD$}=SNpD7IA6yd!aMI-r|@tX8^@mLUp^M zn>JD({wX3ZsV*0E(H*(R>Mtb5cD!^O_b%+P^Xv+AG+-`6u8`tOFnY6 z`01%gGq3p)y_2jW$9t?}68Vpm|z<# z8cbf~G8Gum>%!r+d^FC<0z5}j4MZi_$Y-i5nX|)uziuuBQz2}v>-xjL(bFcPeoxx%VE-(q|C2tc0d!lYM9JFao{5G2=wLX#naHS3Q`50w z<*?1NqSLQL+4|xO2=YTu!P>!T0U}QS?lQuBmwf;AZUI2mOS#)42mO;h3Z(jHt?DLc z6c58cK6)=va&wH74J!`NNN1-96X#R5d@3Tee5tuU?}5Od3(PfM~=_ zidchdJod0O{#w~Z;-tnRG}Bi1;CKVGSUpVCB{`k2qQLKE1@qN#0?Oz>`NXJ>C359! zh9zRMjHagopJBdm%{Z$)Heo)8-}+uY79q8I_S-l9WR{0XDt4m=>ln9@DQe~e({Ok{ zKbkKQeF(g?coF?@^qw^A*w65uyr1c9SF!8u4g43w)5&F|+0{j!%9vdY`&ubq^p-GY zw3w0DV~qoqVmx${RishayEz5mw=?rz%3Y`fKr~yCwAQuyOj#X8A)HW5<#4l72ecs4 zsDQjFalGhN8!_VKjHwfd*|^UEPFZV`d~aG$f}_p3JkD^#Nub0Wh#07tbl zXBf#~OdykD*`d{6W-;Nx^8{#4lr!CIcYs`I(WhlFi!jv-gsWr2qV(zNX@1S#zeB2} z!*EgqV11gB*#lx%GYni!kd&t&bX1V&;1h?~B5noU)5d=J2#`ZfHY5sp zPSzLXbNDvUSBlKBBHBE+fbnHGT2dN$$#nzAhSIxC@S8i;w)}^pz7#HfjosJAgOw zKsYIMcR1NvTSG+ISVee>ZH%(#?g0k=9E-wse1~x)e04{(C9Mk#tjy@ox1kMNA#?xd z^6Mw2bJZe8Lwm>tJN=q>qYx@Yv4^%R0I4ceGDfMR*3?cM1c9V3{b0QMB67uMnx(;X zIRlumvL0%49&6H2k-q@w5=E=m`_crih+>@l3re9Xm$X zRON&n3iheeDh|P6u|rg@5$Qb(pg>aLrPlz$Tm?gr${Plg%GY7k{!puZjo8G}Kl^;t zRG*L{M`m}IhCryb!WjT{g@eIHGHgAD&_C{?Xl`DvrnZi)yqCNtJeMdFHd!`;b|_7I z`VRFtXzkY%zR83gj~+RMW@aE^Xyk9>X|>7hEn!<#v7p``mK>r-`io$J*8W@A{cztm_$mR?EJ1fR)W7r4Va*Feju0_S8a6zbStc}hx?iM`%X zUFC*M#Jvs*HlgA11n=^V$8VllWxCox>*i6?T86n98&_;pKgn<7J>$SQ}k&{pj`oAQCEa{S&5|K*b4QSw(xB2{SlQVHp0MFn1tIcX zy_gqtCg8fU68@|!WJYe%C&)2d){KI;9+HyYq~SPv+PEpxl#1)4`hV^GyI_Q`twNa{|Kb0*>r{An5XjB`x^wxq_xssi4}M?em{P1VBVZYES{$2ouPo7IJdn2VV*wAVq z)zTvAU~qL`K6Mmxv8d~)@6b=bt0=sSLuux-9VC9d3R3E6sStO+vETve7g|n@-%-X3yTZ+DL%mhRn!mQ)7xjze6gMVngXg3q4Vx90_?jRHBtZK~yjr6=e4pG;j z%pycGXi@Tv%te^{V1(IK|9=25K+nHLT_bovpOUy+J`5^Zy0WKM2#OWObf`noTP>p*pAEj+waOiFPXTxu4!9?v* zx&lX!Pwc*Bi)yNF+7q44OPRL^_!B!c@D=-5-^)}6$JbTGZ+XGi?3K!B0TM*ItLS2c z0?bM!3#AV=Ei{D%b?c4=yV9C+fzTzTC9KfrtL3+nR(b}k|29Y$xZhIB!lZParKYu|G56g#mm>1Z^#>2Gh1rYNgJ87?}lU|eL2U#LaGJWtsq8U1 zx}pN&LBm$N?tGd$jEl&0gq`KA4o^qO2bZcw=hO1EFY=t_o$-mej1EYOd11+W4oGtK z_WJVwp7Lg57UjbkB-1ku^vawxuI~(X!Et2opCs*0mh`~fca7*GxV1FMR_1I$u9AYV zx=;E7cEyN(T+|#?7Ej)Ygvs1YucVwo_<-Jx72jpysL^R;Y5dQ`4^ zg5*rd+DMN2ZGh5L$&8>CDl&F9oub2=6~;qZmd}+P&@21a$N|Z-ji#)IS$=Dm(r>O9 z&7;1wlGYp4!M0T0upGZI3Z8QVf|75W_&&D%tj7<;9*h?UKcd!a17}i-aCE=b67}=b z{H-W<@1(51a(EiaE9ZNZNPBKr5R)j-o`Y>iZ>sR{wNM@|13?ef_;&l*1~9hWszzL> zuPfOEPoO;k>+8#xdYxzq6ci#+S35w?*@MW8`BqU-{#Ngh-kxaVK*FMde7me8l$8a%;bPYb$vSJy_naH9tpj#4QVQ~6#dDwtM zSXOVO&>J&(VO`otQ&;@o+mzrWHyTd}=#hX;70$s%fh}eQ>J-k_-6#H*yA@hX!h~unGf}>c$u#lX%a=itPsy2X<2kG?sK;JQw9oW)y_F>&MHPh2 zLS%)EXGVl1s>wEf0eNc}G-*|_V5{D~OWa#Ql6`B>guNBp_I=WOd+TAi<+qAelbGeV z56momkxsAiF89I3x@9%ZX&v4v$Y5Qe{|VC_RYgbGY-qZJG~;Y`gq23r@EDu@ z-*C}<#N?D|W{~Sw@F>ab2*J!^Bx-K08HvF-mz}jT8xd!LzfcS_V-Y@GQIX5-UL&<2 z%sL?1iw+}7S$anS;PH$=1-oSt*v~#vxPC&n!cH}iAO&5r!o);kaCF>6Flot7i5`>1 zv@HC0vS65xVmqZtd{%ve8>jS1tBKt{ffD^W^{I)Ry?cGqhjq74aAJNweM&M?DVAk& zTV7RCtD2L}0P|?;KttS!iNVLfJG+EvBC{_F>0p+%{OI1blaMPF398#SlJ zbs2^yCwuU^udgqj@?04FPH4X6O2xu1^mWIU8!2zdS+F$DT=BJfVix;c_aF|8 z%|+pYnn2SPwM)%`MrN)TFcDZoBF-&#jzV`Bux^B&>IXY%_NOYITN|COH%3A2;p|2` zJ~(D5+ddq)C<Q+vn+6y}sQq29~ZGS#r zkW1lGL&9v9pzYHqWP`41%?bVbIWRu7P;tCx!%MYPS~ZEQFlKNjzR`K|b_3(Hc~GAt z_K+PkMo0U%SNv|!Agyod8l_Vdk*?7?W|&T)IAj96hXs&4t`(3BSpt7H14Hx{KX5oP zsMilQ=KGGlE1B;_Adhy!;5<4g0Z*(G(kN9P;G(Jjkw*gijr{|Ip1Un+!GK!4CN;AS z56Hi^wv3Cv{!d^ZF38j({469L9|hRKvLbi1+UgUJW*hLnQbF>j zf|?ICqxo*GYFcZdoME=o_DLI#$^^R@n(%$0|F=Dm&&nR@pgL*%`m*SY;G+;ladQq zaurhmtRcv&w_mxHGOqbs43psahGH0@O>f1xYj9+chY~Z)SZ9S)VaNs?h-^cJU{?9! zHH4cj852a7%sA92QCzGGMrd8rY%|U(zJ~yL9N;!m)O@4riqS0&bj7xm<0}ZwDj5iI z-8MzdOZRf37zC-~8R-TNB2n6~(i#2|nljTpDf+mgGl06@mh}!J;|vsLrD~EDoCWOj z*h0k8gZDf1V_|x%OqWFtl6YAw5`-UU8-TASIafwIQZqtp%lu&C?YP6#kH?+hL4~cs zTm9E~=!tEBp4)8WTdSaJw-IKY2<}$)GBq)1uDq={N)i@|V7Gd0vy5r6fOdDJd*Q(w z&pRVuGo|R7T?H)dW6CQS+uO;2ebIC<@^;&shnf&r&1FvDE* zvv5zlF<#`gCEgO%XN1TDRMB*^tTC{kje0q$nfhC%b6>qVE$VT2F-rK+mi)RRyvTO>p z8h@o%HjG%K*nXbQ;C>2?s3qOwJ|>pVHc^^bbYm^lJZzj5S$*~^S`x@|A0=IWb9+=QU=+h!GVZM?QNRC#DXG3VGEB0)af?KJzyTXj?rrrV1ukpJFs zCTte49@a0O{{D}+O^B?>AD==zs+u7~0-mZ5%q=Yntz|`xthI!RA6nQ4uj@K(2`7t` z@8AE2mK(>pVy#%Qnqh{7)6`Dk5IQy74N_9}!RZb@Uv6d5Y}re8%TtTbb)K+RmVzrl zL0T3vyCJ#xu@2(Aw8hgNxT>Y!!cOSNmX;+KYmAq#=p8mUsNVq%z1|^r`N67MBH_?c zKG3iBAv^3Pe8#}+Nf=b5&ERl-#4|>Z5$wLG^MA_6c8UC6KkHx+3F#1|pG`D!ts(^C1N7gzm-@dsVZ%=kCXCj*Ge7{2teh zbl5yUzzF;;3ts;T&iPLSm|LS8I0rcu6%av5?@+iTC#v&@lbkC>pvG@zy z&|3!SV7#DJG?%KD^?WaQyluaLR`jE-|RJn{3-#KcOWz6p#rHIi?7Q|~$;DC(%hf|wF^E*0g z!TB1WDFFW)kj+x-iUzyVMZSbLwc}{K=ryYZwE&nU<(M1senW*+@0wy*w!Cn2S+TO< z88y%USi^JwsPu#XFu(oS0&}6~he9b9G8kuRDS>&W^?kbloBULvHU){X?OMt!sdc^q zie|OYf>rf^IWsVC1t@9&7q&n2-LY-&bd{-@8AJkuVTchaNFimpq}k2T22ye##e2y` zPUxC)p<29x%YoS4&}m=D^;hW4jPyjVjoTC4gWfu5qYCXlgj!~MB@}FuTdMdEk3%UMg7z}o=dx-e@bM!7}%VvL9&cV$+@bw+7_WL|X3+0rt z*mCiX(R@5WdNAlz?0NoI()#@=5$vwmkqon~M@$i3r91t47(v<8h&H%_I%zG>&MK-l z5Q8egBRUuv(v4zu_1(Kd;Zt%}%G|H7x%a>jqftaj31CK;O%H88Z^KRmp+&i|fhQo$ zwLu5u+2V_#hye8QJzIS7doyx-!78wWuWMSfRa0EE`gM1*255GoRQ7%k$0&p%6NJp! zH7nqJ|2YnzmykSa#^sH)!A(~($AB@FZCKtE;4YrTr}4|{T2^$;Uce5HtogV5PqbTK z8G=ON@A?EmPBC~C7I1=E5YPe2U-5#WkGha|5Hy;D!un};Ch&VN_3mC#Y%7 zLAm&FvKS$UMfdaha+gH3tmIa6m`2SJ3QGoPYkh=@ETy@9Obvf@Eg5`Gw_6WLShqc^ zi(Vg$8rzmK9(-rJjK(Km|GDrv?k&yX10YoZBPnL9UVUV5TcD`h81yI5eDqm@YWJ3w zz!^v5bxcry#guHS%s~5Y>COUBE_bbop_Z~RQG5DKACE4nSY2^;%LuKOysl`qb7+hZ zsIh}RtGCoY8&>VWiV68=Arw=Df;YPoQWmx{=c^8*>I)mgvPPp=HD5P8XHUFI42b}v zr_Px5hdl25zosSVd2qj;Zse9Q=O%!#i}W}0T3M~B;6E|L0~TOwO6~lu<)w}mq#Upjwj6}&c#8~I7z~_Qd%SUKN~MD1BB9& zYJPR;if4}QJH3EG8`m>vx1X93T4RYb{%3n++ptCJZDZ zIN1cDrRHlDr?ug{w5jEm);yy{F^ZmMp_R*uZ)wGgor{FTe0J!G6olt&TgqBXL%67X zactCxy@G^mDTYkIllL?S80Qsdl|n_~0O3BcUP@Wj!{E8*EP77v*fK2nAbyt~QrMp^ zF&H#yk(0HSGa%c`A$kQIFtsE<-m&G6)5K4szP>1^Qe*w`n{4hUnSOv!rTAL3;?|WM zFnt7HZv_>7)~i<^v!cma{#7Nnuf$4LLqY!Cy?7O+K;|wati5?i*3-}1O0|Xz+KU(v zvVk2r)D3al+%W&49rL}cZWOUjWTe9KH}*-e<$AMFW^f8 zx=ynhTvT4hmN5ijO6&inAAKcq09dy{Jywl3>s>RFa^#HAG=7BlFR-yck{91x5?ZYr z^zpyd{K}c!tp^X9efxCr?AhY!pc<5u4PS4D{ESCK=+O}R-^~#E&(1E5@Q&V=#_lTX zch;mhs|aiNd?klsjZM6kG~C4=SilSH&Y#cmM@T_~wb}Z%_cz1nL7yNs53y6D2gb(Z zjR!X7x!xVS)SW^5=R^9@oRSr3-8-j@*@?mpZcHa;$!#Ia2U2q(9duw2ufQH@;&Ci0 z0N8Mi=BA!Nl1iI&HB_e}%w4mB^0Bz)t`!y{*cFJ%EqJcc-kn<7R!;p)0 zW@=(q%uBQAl!-)}>}amez87~8J^&j1awq7PXLgP*3R{G4gmxD?z_g={l;zuPQ=_Z; z5{)S!%?C?{m_y%I4I9SR4ThT!E;krVKgv%EQ)gw2mZ8 zblO?Ud@-;_0`GA}Vy$^P-X^YgQ_AO80G>-Sf${?5-o{UDm{4gU4% z><9Mh?}Z&X#_J;2?_WG;#$+YMR_2eK^tvw-+Ywed@YX%E{Z?&0Fw7u0f$}O!#W%(wEMUFaGL~eOq z8yl&|rpJPpdELQHH+urIkSRFK+cG9To_;K?qn26{;oGtR`3`f~txN=NmqpuZ<1S_& zGsbe$gt24gbJ|>iA1hWfF`R4VtlnSze&24{Rpp@~t~HVPo?f+~0CxguVh|3nl2r&& zhAVRV#qT$#7?#zOKPt9v3R>x@t`G)={Af1IR((>Rwx2ei@=tf4cIQ71kUrb;qTn`! z(*C>UqNyQCtf>^q=}wV5#%>hZF-nzTxaW7?=iK|e@jh?8&&vN^ zo&WiKe}4zKQ6-z2T~%zwKStD?_j(BR-K2!uJ2O?WHCIp=3{bK+ROH2Q6@e9I<#T1> z_KiyS?WHl%8dt6qgc8*(=OrDA*9~>aH~ozVLJr$Am1{+TFm#&1fF}XbV{VzrneYq$ z4%t4mTHt`>TT?Psd+`h=DiKgb#`E{WB08+{S8idaD0uxx5R=V%MhfGLz|rd{v12^z z{idhvY7Ot=&C9WTjdYdgBG4Rcw);i%^HEXV@IeE{r)Gqb8@8L7kRoD3W30+hm~w(_ z&@^PP*28`u?P}EOIWQbIZ0E2IU|jSUE@$2i1s1iAAza>5u_12MlAG^_xndBx0ZT>q z;Z&Kr`%E?-Kf-pCs6zW3l@ z(~^hAL~0lVht}u3cn;gm%Xsx-H^f|#OF?91$G~OYC_~-{kOah)a30;I3Agm<(!W?&P{`2fE_1V0pD%57;!sz-#@Y`te??Ot0}tszV_gu0egzhzPx)KEc+ zxE)}e)y&NkUwgtHux1Z|Ef5K#l`NTC{ zE$#oa;kUFfaWPa7h64NuQMuYXEobCzBNh6foaO1{hV4#fogFf6!aTVYCl2DSyD8l{ z1yaNb{C2X~J7C8Y*zmD?sH{KD-=T%DMA>WabZ@%cUWICkDTWW;Xb;Xxp#2n7t$mDk0#d&767O#|HD!qE74ng7y;PU-OOclXa>g!52 zWlFHZ1zPQCK~z&(YkWc0`gs#>fM++X&e2L09=JWa76gGx(3S(Xzf5Z{sTGl4rr#Hfy3A8U(`ea_`b1||>T>}F zZIYiJB*@Qzcm=5z0hc`#I2F0P8;kddSA~c<8qMx_0ShdIYTR|v4Ls)JmTbvr@X+ZS z#t-MQp|-&pX3dyldP}AkD*IRlu+>e>#g*Q2UeFE9GjMIagLitfboOEhh)eSVS=Dx^ z$dW3Sw@p9mt&($y8i5yX=%rL|Kj2rD;i%PLxQ8)Y!PpPR_inF#~l4hP{9 zBPWBXmh;NW-R=Zqyb!;Qo^fz}-0+6UFwBo!k>`m`q^Jk5C=k3I3Wt zC|tB^#6>d4*PKN&zFO+%Fa%xH)ea&e_wZ(2X`qoleXE#qH6ZK(=8C&OZW{m}o4#|= zKVKJmv;)kQ8wl?Hv8<&iD(@UXF7xh`V;?1ybhMe8wqFURiQ$2Dius147u zH;hJ?q|e~kTm_D?#2FlT>)6Up;Af9?kUbJVQ|ucqnvYlnCB?L?m(=wucnC{Ie5{Sy zfO<`C^cp|Vcw8#*V;4pWA(@I!u0B9TMLw8hps#+*T*jd{KgUD#eurq> zmj^}wuH4k2RRE3OJ$*T3o=Yi@kQN{6{b!wYO_aIXQAg5p#ytR$+U z@39Ca+-JNO0h*dKx1gRr-C6?7aa+=yLZ_I>!@h=+)|;8}mm%mMI4V)xKUF@4s9h(f zxdEa!-KcF1oMbE9_gs;aZ|oSKn0wy5eIM?+P3i~yO-z^El>q?khhvQ{ zZtn#Fir)%YP`a&it&i-<|!P+T|Fy1m~2vn*b5g60d{%i%dTyi zNTUwv5gu3ay3*wgX50AE$UtQngs0h&uP;J>Llw_` z8~`(Sm@W;^$2GxmF}NeS3+Xu!-I!L)0C9sKZZ}0-4fS!XH-Wx~`iSTxwdMh6P35<2 zM%XPAgpYN;_94Q!Fn|;0D2BE5v#NnXFLz4S=Xfen*uAWFDyzp_%Uy3E-U(hnFC@u@ zrRSEG10D)YD@Zqh?h78AMi?`@N9C+9FQMO~E(!-;(Wy#n8KE|C8&==RDpa}^M#Hy- zTfD!>stFf-b1te0+W=fu<`K<@?b$Twx(f7YT$Op*3lHnMu);>8uybq3qmUAG)?SC5 zY>`;uLr%9xvVsPFGytL(Stc7X3S8O=DC|El#4Nja4Yb%;RK4|OP^~Z*79fyD`Y|{P z2H!u{cvA(ya%g4X#y0ZKRS3Xj^HIZ1SS-l7qRRqHz+Q)nE_Q>ld@xQ^tdKNN1>K5Z z72C>NXU=NsF+@x)B^Qh;7L2_aXCM%JG+I%v+v->5UM$RcZ4knx!<9D5pH~bgAisLj zD$y-*DA6=N$kFY6!ylrbNzh~RBpfwwSxy5TDl!93fPfW?-C%>^#S;ONRmln4y z=R>unhQ}7~_7geoK)T(P3=((DFBW3S@hdoN0tD{^vTYxOZILPw(A zyT+J$>XrV(_IM_Dfh#H=;eTdGoD*4**K7+dyx*Kfri`RXVT%960HkAl>Vi-)sbDdQ za54Rb;+kjKa4lrqwNy_Z|Qdayu!^Jt3h9cZO@+oi&sD2s3#2E?IrY7#=UzcAP?+#U0k5irEYT z)odYA0b6gN_q62O(amFiuTtiFTTbx8-*au4S`ld}S5SxrjIuw&Hu>x@iVj36ikNxa z@sf1<*_-cw{JENRSb^JQy$EviM<_)13|8XT7p64#9~R^*S(##Ew}{cI#_TK*Daged zZL~=6*}uUVI}DWgH507jS+93v*6j{T;(^Q1jr!c3wQFp|eD9}258nYnA=Srl3Nr(?T{3X?_z55o z;OFSs>q1b}K#(CanA@w%$5z|5fezSN&a^~XQyF6-z1ef}pCE)ReG zXiQ)~_P)h~EkG37;OgxEXfZEw_|%Uk_X`wQ@L0sdU5T}RRt!LSYCBO*ztz&Zfb1A+ zOBAw3!U38HQ>Mr141(i6(rsC=8L6ctg%s;^z#6>}gni@+_AQQSpw6_hdqOKLikWUo zKaU9e^IIY7-`eFIX{dd$mW19)ZVQ1!IU&s9&Ru@Tj%>NoQhP|xzP8NB4W~kL$+D2y z4a-e=O)N8J-*ENai}2bvsu3EfA>l$R4s)Blqz8~IIACj!J%`q)8?%{P)*FgZKv;Fe z)I}0zi#hPsu+VIKtr?*E114H6$QxO0X;ByqTqCxJ4PQ$}1k0FGwA#&}06cfanGBjY z7DA!|N~i+}f+++Q7RD)AR`P}kk`;}rJtW>XBeP;OpJP4_*wajgTSyGDylG2wSTTQV z%dOfnA>X|qbX7B`7^{bT$wu9J1dXm(1m=SPd_66{&d{u8EdPQRfw+kj4;*oZr*I~7 zx9D!0YdTi14)$t+nMnz6 zS!GIP{a`(NVnpl-;|Lmu@sG8o6JQLq;G=3L>hIdKJ={b*u$ZWM zJJ1n|7%#!hvm4nUMVtC<(-X1PvmToAmB>5X`&>UVje>SfB*ee0d z0Ix(y1OMg_uz|fa2SQPB>fE$XwvJI>qBT~~gsw5J8O(F?+m480#8g2`wb6nH;uxMO z^TonU!s6zSFhkwjL#{}B{dP|xIvd;@=P3vtcsY3-3O0kC%GNP#3(>(qnW&bOiQoA4 zesEPF@+zoXz+j#7AlO|3MJrulzN#4X$#&nEKor|#%HC=U7h;El_|jV_o@SRolN&8k zAHUvmb8T&lx0gIranQe7I`7yDgYPE7?R5{}Xi>4gZC3>xTrC%x9<$arX< zT#+D3X#cABTGZyb`=jXHGh*-euk?d}fbEV4l5qQ1H^B4i5(6lWNfP4;?a7^l!QWn( zGwwue9A5to-Q4Cxs@~HBHuKt-&$u{B| zQF`RcmM@?6gzjXjnpx{9_W0~f@7Oz56#C!$aa*+*>NnQ(gn9(lke7f8iPu?P$+9%2 z`0aVW-?oxIK%7}j*Te}k6~zHZjmo}dW>^OAJXBew!eY_PS2^U z2a&QB&i=w`dQQGoII!+qaGur?Z99RquaDbW)TnST$k|0vZoKz;nr8}SXrf&8#-r2b z7nQ^+;6U_Yn}5_>k9zA-Z|z?HmGxFnvN^DePAcje*}TpkxJA80Ly%+XBt}Zm7U*d~ z6!_g8-7N&>0&uamR`7TBtzTC(7*g+FZHXWkt!H+T zY4r)zOYUD?n~h)Y&xJkvQJt}xl${2TI1r(JAk^=|>3;RGq(bZRUr}C+N7eh6T^6m= z1F~6E$EQBYuQpsH+LcGfP``RZg_g{3xolEKz{J2dHA)DWWWQvl96yrFXh}0(?@WD+ zR0SQRkeb$wYQ;y8J-`Y7fKFOB;}0z*hAPtbuPNy4d(VbL!@E~%p!o1p$>%TmJR^F$ z*Fo{U{cCas+|+Y;pib^DRuQ*2J2s))$0l|VPP!*9{gv+svV{Y8sQ;D zNR)bMDW06OnW5G8w6(jt}6`qu&HcUq;i{*_ueZBplZ zRn|r-sZAmosk%0)vReN-Xt3jz*ZNm-CALWe)2-SzTA6JU%{pGiZ5)(uoMb{(w})xw zUX`~=WWyEVQmel0QJuo`P5%yAms5hN?Uim$!z)xpZvWm$sj~hxQE{8v-)I%O{&gQ+ zZ}JRIty-6?+jVN&MXJ~(5{=Zl8?0)VNII?dovU(}tfVB@!b_w$wIpr-dMu6Nej$g( z68WN_=oXl2!R|jl0F9F{I@cbR*3o03G#}c;n(5c1B*0AA(n$Nll@vU9Kt~CAd5$UK z6v9u!0Dd<_(#zQA%8YRW*)DDkL`qy=lDy)#kSyIMe8|eAj!yGjxO{7`I^!M`Z&fi> z%G5y1!R{N0VFQS7Hf*aVWmA<>rL~!gz1;)G8k1{eQ|mChGi@KhG(du$P4m3y!pRsH zM~j`2;(V4PE1-u%Zo=b|n~CYSr>7X#;dz0yI^n}ormfaaHz8pd0au_7Sr_gzQB@sH znnW;>rEG**!+8`1-Cp65)cv-()b7Q%Av}rBu(oMfSj5U^N*j6K^O>z!Ri(0l%S6Frcyk;yZluon$Re`yeC4NOV1?+>v|2z__! zGf`4V?sbx+8&_KAFdL&kC!$@C-jIx@)0vi+W>H%N0}H~~!%MoV7u!4~S|x!KghN}q^J?Q+ zX;&h4lVk(J-@UqiADmPp3; z#IT_Uq?QZvBIX{|d;;Nj0Eqbo-Lm2ZRl}eV4>_$60X^5#a=(d< z?$GS)L==TF@qob!r**4SEAs|`>5h}0xE65k9%LEBQz5$ZgvLSR4r>JeXT#bOC@#D` z*5OM}h3X=0xifx7Xi;xuv)*8GacXe#mYS3`LhX47Rw6hGG3^L5=vwiLmYOS9JZ?V( zLvu%(UAU|eEPlp|9SBt-H_=ttvuG4P&)K$=H7HN8Wc?5$6e>9uID8Qiw(aE(^Qo(g zL!pfoXO;R5%xKC&i){FW)fk2@>jS#CLbgS~$~Ls3Syh6Vux^58gklsTYzP1>LNg2xmIVG4c|*sLnh5+AZS+2%RRr`aLU9jp?12EkrhnIS(vOiGJtZ-ELdx?5J9t{E^TaggDJi}a+M1TQJTYnc zQQDdo`IV4FnUFx49KUjC?8*eFtjB=KdK8GPsqk3IgdYVO>k(kF9s?3?%S% z847D3=qe_2L}0LnK(1m#5rV)P1-OdtgdqIY2)I>B=VQ=Uy^w>Lk`af!+5@$Ui4zgX zs}Tt)CT|7cu13JDVti!;>M9M$Dnfi8<|+-uD#l$roTDiKREXbs4vq>s>RCrU>!@d? zwlxv%>2aW*Ccr#RMru!ocuG&~NrHFk0yia4jX*mMfHox%4Z%7M0yZVk9D#ItR1m`d z(sqSJK&D3mV|pYYrima-sr(`tfT@L7O3^dZz?ZtvN=Z`NFAThtE(N6mE*+Y5m<%p6 z9av^EsLW>#D06Qxna2QC`fLG}CgI+t0x2EW_^)e?KNMKfB%mYezvS3wKQT97}5biNO5!UbP%NUgMT{s(Ny3gHw+1jBo2C% zn#G>l5`m!8{A?m*?ndS%t(J-t~ zGSwmYp+QKYWQzBK9-0U{bO7YgH29qDwe&p+hyFNwYH*O2~i& zB{U-uMra=(D5Z;02%$bWP?{+@1RpdA43t227&_=);?zS+(}xFBO#=a%3;@)D_erxx z{=(otiJR|0(4PYUf8q+wWU!w&s^3JQp9BZW3N=I#WH<{J?}+*3#LT_&Hq2QG)kJ)4J`~=%#%!a;kVO* zAU+%VpnU3c&!*Hu0!+6B(m;M6o0&t@-p*`(k8w)gfm^YR@mr>*L+c^mFj6WApKvgC zdt`h{$x&A5SgPA&+Trpj%z@KCn)nZl|4k!lnyICtg4HHE(1$VDP>~l}+={7~hv7b) zjLtYhd8o8tjiFdO#eXs>_z>R7Z<)|rYC-N&-ZWRGC$J5Qku*Cb{GLZ%OEqT$XT6#f5XyI%W zJItl%S_$LkWU3D*>?76m7Ma{W+3w~Uyw-XAKAc48UX--52}VnK>kK7xL;!ICLXB`O zh}z;H=cL0QP4j`0b;YrqH~^P8z~=@|TCF0R%u%i4F31NlxVMvlN=#t6T$y#|&?{jT zyl`9a9W3NaaaGB6#Z;890el-s-yqfCKJYo4s%Zt_P>TO#j+!(@nmd4#d09}w76a}4 zit<7q*3gOsBZA*Xtv#>cdHt&WhHG9L+g_w*)s_odV5Wg?;v1PWY@E?Vp7+Kmu8;N_ z0uAMYs|~IqH;(QUk4CMnnhIF0d)k+PiC`>u3F$=0T#t2xV%tq`t@Kxe-WHnA;~Gfm zc(-TW@$5|Yq~X+5yVGr3W-y_8(-@)ozM?__)C=6MwOL^jm-?;6^fSY@Vta|N2>V!4 zl(;DfWcobWYI)2>7tVdjG^4TpFPWG9HMsZ7+iZcS-eTl*{J|PRPyZfy+y5dTPR4o3 zhm)V-bvW*i@~XY9{m5^5(NrE0( zNhw0plcPQea^bT@Q<}-h&3dyWVY0)Hw_(4Sn#}Tz&T< zX8(fDTJSWmAsYez76eu=5>zRfaTKp*e}Wr^|FYa2@C8_aqC|&Qr+6K z7GY`fzK*MPMMYE4ka)Sb%O626O$tJt7Jxb_`1G?4JiS-YX+-H%q^@XoqdI2ws4i!z z7(4fOFO2TCHtY=D@7uP=R33tf~rPy5?dA2RwJ0`q+8hULl(;^k_H zrY~DfrcDbi)tfiKc@7;W87;B~qX+T5Q^(fJVnPT@t}tJU<+G7qX!#>T)le{yCkB^6 z$hKAOoOYzd=;7s1RLshs6B>@T|2t=}unI_%=S z{JyqCJnJSnXp)SsIl8tW9?Z3^7hrOV7L)6mYEkGkJg!MT&CX~I2D!A2v7XgBoI3?YqpsK#Ah&xt-^=K6&R&rFI~);&WTv2N1AgVQC^8KKlF-fmp9!fA+Uc~OhGT7X1zA_gnN z0s2{t-1jSB&ORPr>|}G;Unv0J1OEy`7}z#T(0%;hNO-icuA+ z12oced<-Mv5hWih-n9iJlDbXx{Z&09TbgaSFy{!G2lNn{h)58-)(Jwd82}Kcp@D!< zZ`b6h+8fT)YTUdw+NdHY9fo{js(Khxw0{yN5gtE5>>2UkQyvRZI*=^fJarv~kDvFu zGI9Ipb$Hv_jP%hHCcQ|cJz)q^Gj!%@F&`+fCPk$rr#-~tlZ>P^7?e3Nv@b@gC!9t4 z=_f!8ha=mFGB8m@^N$I7o*Dtx5Ht~heWXtviJSQE*ks61;P9}Hmtb(%*8qn{!0L7h z>ma(G_hXVLO#SQqfaac(krwoX@Vs*6MPc})TN93lo10)F+n@7cs)h|~Rd8F@aYj5z ziN8nO^Fw9Ur=A1-6eXBY?oXuMnZxsxz-7IeymVLDWzZ*!N#kig_{AX>M+|8ZbArj&@q&0)L0 zfY+=TMH8L((Y+?+u(b`bw%`rbFM<%N|HPaLsvC|c{W2xhu$ybOAnzeiKo$nK8RBVX zPIx%9IIv(WS0)v~tL^Yc*6cZX!|u#JVTrMG1kGk`0@td6jjuPDYPNof%ni$K5-g$w z@>)N6@(~9!KFA!Di9=4ouO{+CKRG=Vz?h~PAxKhf5CCd{=K8bo9h=&xJNs;m5 zmnb)oaIhwdjOB)^De@lN-#_QANWVv7C|$mH^g!6CYnp9LY9jsuJ9~J!))Y&rLLEwk@&^-7#mj<`Ke?>* z8C$JbR{uMS(rteF*c7 zcplnidaaMxd{htUGSB`~{X?o`6_*Hw8nP|-Hp&@Rt+8uXt}Neio7#iB3lQ}~U(p|( zpL?9#?brqR%g$zG2V-OXFb~s4vu(^2wM<|oD@Zdu%OwdF&t|tgtIrqY|6!GcWhdC0 z*1QcavOcTYl&YtSJSAuF*myg)Tb859d2W+kF|l=QfmlBP7xlmXKDdZNE@5bL(#rYH zmThQ946Td3rC^TIAkQ3tcEX+L!6tdtmt!o`;VAy6UVoHSYz=!1ch7-4EI~#ZG9cEn zl!aXHuFHziT&pN`1svdDfeGmm1q30!%uqa7fEsC{SF&m&?-1`9-Wim3N|`G{x6(*P z7&2~KeTduxv5(gg{Kig)zVV-MVOg~#E2ySmS`RHe@;%XiWi$d>nYlhX965@ZLWT802Ia zNDJ{8r8^A(GTnBW1p4?maF0`f9w!(|(!m`c3)XQ0prZjKOdxtNl;eb`7KAZ@VggKK4}LLaAB+Mt9tyme47oVnFP$*>$(1)E(29wmiiu{pl-4Ig zC{7Q=OdRLO08D%wsKn_Yi7~*Q$?%A2A$1P{jF_;(5^k9egV;{b-`FlqM?ZPpdBzGa zwGkoP6B{fXcvuG&b&xvi&%6zes2a+SSJ;sO?ig&7Z9^Q?qjNO{wgc1|I(O}~+g zOm2@gKsUWRgqPw< z=3R|=eBMW)*2tOtuyl5*^U=J5$kCo~(8LeI4yV!soWiO;Uf;00Bp!A88gK4Z@yOY_ zVlXziyo@puTQtL|+HT32Hhpf^BKA2&0tx~2rXtlZObB@BRQ;n+4iww?U3ZQDDxZQC|G z$&PK?wrzEcQ>Uip4W{aQhIP3x=MlGP?_Y9tv7M#ib=S?lY`yzD>TEjRn_7rn@lZBP z9P6JrAP5V^g&l?pv?%N*HDsGTOap64A;BG(F^@U0nI3+SCPxu=6?$`j$sx(K7b>&W zkCx0bPzQZ3!=4Z*AaRT%PT_o8hF~OU@NzKKk7K9|`Mh5AY>3Z!i=tfc%&%3!$g&w}Ske78LZ_p`ZGU5D8JA^GPar+^&UzEm+1%(9oXi4+%M=Ah zB^N)xk9+UAtG>rWOlwSutKAxJPq>C^Lx^Vl6ZjMy-kqJt9HOOJG|w)2kaM^ACuEKk;<~uA3^fGha2rb0{8hjqC5|Y zqX8k0T3vF`nrB3_o4fg@MyK3Y?QVCVhL{<@27L~>?A8`v+;|f|;3DFI4b>O-w(ws9KDQ}1o>JF< z^oJ%+YoHKNv%k2l;1sgUJZuZ5wJ@k{Tr@|ku3mOWex$6Q0&5dc9b7BBP* z!6UZ=@uOPA08)o@29DE=sdpPP^IPK@~ItLV48LgxN^XVb2p1+q?qVb zzC2QOO0yGB)9inQZ4$0rk%W<6K62@d$fv^-{c2|2~gmzNg$XYLH&3@pz7 z{CxU2&~}))1B_eeQMv*JDjlO_gc=&n72*&^)8Li%dC)oLEDvP=GPJ56gS3>bwKyJ% z1dYEkKn_~A4xrfh4)i5B5by)3rYO6NTmF%%^L`gM{+&q1QZw@|c`mU9E3?=?kwVo8 z!!Zxlk;Uy^j9EDrS@~cf8Hx8M?4eH0wQ@Iwy~97McSrI4vt})%%@RRsYR8)REAgo+#*f7Ho>drzF?Qo^eT`05g~B3tHH~g2%|P_}!*L`Z ze<`0pi(%v~ph>jg`{P6t)!YllmvV9MVF2WBi-y)c?<{d->A(g?#0kjL@9?LSX~ znOE)4e*K8iSWWgLlW(xg3(3Rbz0X%P-QYc=O%oF?uf8G?T1EOlIvm%g6SPP|Km`q` ztLgMx`l^D14tc9|?-4i&On*TkXKfZZ$hg~cp<@s3d^h%RH>TG)m~NcPZ1r{?dFJt| z&oFTz|%7#zVu)eJqJs6!xsPcp65nOd7>tN$8<`!`3i%DhvKbSS%hb8 z#~FcUwl`R1hoV0^7@_~~%gNT+G3gv4yd{P%2c$h`3vu)DUW3fQfcH>K=<5B~+)xfP zD`VquduM1vM|dbWNO73N1xe?q_5z-9+(#sbaYSKggS;#aJZJspSjV6vgsCHuVif4r z^>y7_A|KBBj-3D$jBxYrx|HGd?HpOtEn-ui$qrRJgpqf7+rDTeYOizzrKpMKXLf% zP=far`377q*j&{{9CNa~Ci^gZr9BbxSmUgWVRM9#NbYR-vRdxg(q{~~O=7He2d7|; zuK(AZ9o~&!$aMzdc<`jw8K5duUqn+|$`ZlN&2QWfeTKj#9_Y(VYui9cusq$`eD1un{^{BL_j+`NJ1unzhQ+ zq=Hs^k%NAzwD%v8bxM}f$#%A#5-1+|HfD5Wb5H^P-+jK9@X1h7+=-N17@!%MT%=~d zl5r!a;Va(9Xvi)nX}m11extV@?`bffE}xJ0J##|AtB(jAm**R2N-QcJ#5#w+BVDt> zvj&FM>EIfrro13~n~u3Yhi$mwj(Rr$HmbvdPKUZ?Y)3;=;0~ZO#Bg9AM>zMPGV{3a zXT?cOKj^O8N#Xfl?WS9+y+zE2J}(c6h3DB{(zMin?*mH~o$r^XS?QZFk*nET88m)> z)($rtV4mNZC~Jft7?>>8uSM{#IbI;1g(&U4C9i|ZxiOJkmvjW#{?n;QW?ji2 z1{JX4dwSq1tBSxW{VbjAjWA%WD%RM)l;P$(0f?+kbGzQsKZ!3;^t_ zS`_-ZuWwQHK$UqcZc?{dy@@yW}tPW?yAQx{usOdwq)EKkPW7KtI@+M0BaCVWT#@>XPH zgGE(U2JBX*CxN)fJp%u(t+z{Ew|84ZLQyO~Bx!ddd|UZB9ZXR>@wMAm!2ou}2tsei zM?SOMD$80%)q1pD%y<=nwn$4?5AG2zZZN621O27p=X)YH*H@3`JE2nZ(oC4K&3u5( zz$QzqMoay`TATGrD3Q|eJ$Sr?K6?LiA+vboSdY};zpj69DmuBH?(Y9s2um|3?6fBs zUQx=8QbPC7lF`rx;&2z?#&_}D64uQx{T5IKf{_4zi1hPQm`R!8ENr-B=Xqz7I~e30 z+Z$~*NY@M^QXJ>97NeTfK=qAuc%;ax(ff56y>dwZO@HhV1*B>eX%QU>Br^K;aMWx z9q%cZ9t0?RXP4F&}Ou!&%f-0YANU3AOg6<@I=P_iRhmHWZq1FXT5Rwtj z?MxhH44=Z`Mw5Fo+xHfFeQ`chpoPTAACS#ESh<%0<)ZWaN7*m>Rst^1p|(55J2wsF zPr-*x!RmiIAr>58jx$xd`3;eSMtY739(qK*nU987CZv@1CW}$K0w!~!(Er(+a{Df~ z7*#l7;ynH7VD5&oyy8I$OQCFp*=Jw+uqdeC5enV)i``#cG|G`4NM}wUv5gT=l3z2Nd^Mwn(OjuRqiq2iI*r3$Ycg=w9Af2^EauU zuX~$ZgbjT5RFWr(jkK3(i8Z-V;*SF_0mTd}ho=dJ-MAb#unPSt9(9Zcz5Xrhr+pja zU0w`laV9PX5elOa%nf3m)fh0fto!3+;HTx@B@y?)IDFDQTK2OL!s$-tvYMeyS#fy; z)E0BnXt1%)TAZBs^l~b}K8yuCh5z*9#xa!L5UB;8S%m!a#d%JeVJDD2O?L#A2yd=t zUs-NYu>cGtP>A(k8jXuKO_#M48Wzkjig+Vcki8!+xZCT(&~bia!7E`A;V*=`h;cRV zn_m7y2+$PFo6Gpq9AHz`sL)UmcqF(1?+x+MxdxFKZbXV%^87K*A1F;c`G!8sVzEdL>Lx_j0<(S$?=|mmlW0E}< zi=jSUV5RS~+{U>HJgK*UGH99_?9Qu@Kij(B2Ya{6f-yY3eg>yEZvg`FN43g~w!|h0 zfYhe^cIomCEz=~e=&9G;CkcP3i@T=f&y?929|Vi+SdVss#m5sBk={<>yZC66@~ZL% zULj6w;CcE=jdjByD^2L%qUTn#!_M*z`FT3$RAm2uj|2P!z3v-Ea6J<}cHkq9;NQtz z57;kTCXNR-I_28Q*E{^${eE%8emK)QlW2uLk`W-n)FKD!G(Kzh_f`rzcSVd-o0ksh z-R=lf*GOg@q*>W$ger;Ch0+5>x5k(Zn|$q&^0d_q^C943ejm>lst|NvM88J?DB7qp z&gl;xMhsQ!>!u@)^7$K&e1X zm1ubwz)rvtG}rD}>yPF7#cJ-P12SLQ10Hvdy!24;dqhLPK4qfcJQ;4CBzNXRwlkk$ zZ7}ipZo7L{?#6wukhvEjIV7c0t~2S_Q0{~Gz6)+0xS~b3%A95QqJnj|$sd94PKt)O z9ONzst*U~hUOWZvJ)81SRxSo~?ZmZknxoTVS&Eo)i9?#01GkKdHkAT~IPmGs9Vk7% zAln#+5ILUFNBP%3oiN_rBC0&OQL7G1A!-)US$qP9E@L4%|0M(dLnvoq2E2ggJkJlT zT8?OkuEDtDi1lDxBlhtc%kGt^X-#4%Hjz^+%o?Qbp+v(IBElCRT6M2xFJ7<^!+JZCQVo|n~PHLeKsMn@Wd!j@N8}2q2MMEws zS7-rUM9a&eWGbI=U3;nGS`vA>tkWSibFL;Nev2Yl-X{NEv+Qdfb|Zqf28@Ir5@(z* ziBBo$iTq>aM{IbwaKM4(@q=XJ82zW{u=x^zt@yCqSnPkOtNbyS7>gBlNoJ_kN|e69 zA+2O==O+u|p5ZP=>#h)of@y0t0f2|5?<{OQR(rG( z{1M+FHB&}DCJ&EZF?bL1LMaC!UKnF_IO`py$8Y(Y+IIVZ0Axi_0U2LY7^h%E5~Wv162_j z%!R8YD>fJYzFaw0=@+ZY7nCX`U|JJ-eT{-CgaKVuzphU+ z#1&iHm=6!KMa+_+AuPo)GJ^yPT4~5(JmY4C=fJfmBejv4P}~bQ(KWKOI#i|x0ir=> z+KaJ}PHmf0gf+0IC(l8PnsoW)+x$qd)=7tL7y|xfV|Ea-Hs^=Ymd3RJ7{Fi^kkpyu zYWO5?TOj8A_9a`*XOBJ;Wm^Ar6S~Kx+_^_fXZyu)ZGXWJXlfWd&socDO?8 zg@ulZ+Kr?tdH>&V98Qv;FD{vtPuuNKR@1QiP6XissX6v&SVC`Lk_d-1#6>hNteNU| zM7|62h*#>qz%zLa`I&9>=X?rN^14_u#9|UF+W1e^PQ-4JmpNb5XYFvP@cqpe^DGRK z@$k2J%ye~I_5*qO^-9%|03W|Ty#mX9i8@ZUH^(oTF=@(xv-ZuuKbu;A@rJ8bF-~|D z+gf{4wv*R;-_SDq?->*pG@;?dfz-z+333sHQYMXZW;E0)_YBbYAT^j1YQS+^|BmrV z$}o$}t~Oh`>YeGNg#?fDY~Il+i#BYGCF`o_sLpl%;^`9!#`Ni3gB@wB9djdrf6=bo z^)03KD2E-a2ilbO6AVDL?lH5@%(N%nTC+vx1**eayFn8QEpjuklq-Rh?Y99m*jA{( z0S)n40=llv9)xi~7ne8dT@y2Ve9!;omH4k0VN^nlH)Ayq(zi>0l`OFd^m*d<;vbdx zC?1+`)btMwR9?8j7!p1nTuw!REftOC&7|$jQFruv zCO(i@pO7EnPiKQ2su-s!=UrPW5Uc+0pU7=cDIjQH3k#;F?s}BX{*;w_h)y#8EabSn zLH^A|)I*2^Zrldg0tR9b3af>8Hv@^S>eyEd<8o-vX8K$TrWc~gV2L7;G!*E*V1q+* zEV`@2h{D8k6U)2UHO1EkU{fkhCph0~-%LXZw_Yl6icFsd3Tuw8OA|<@#6Oi*<6RIZm4o@Eb*(L=kr73tNabS?ZR#0Nj_5^5l# z&EMp$aPCFLtCscOTkJ_D(b0+^5U&Ve=4CGmF$OEDzHfcCVhuv4GV%-p9xqSPCI+ih z6&EUFgUetwdL@*!8Vyj|K=yT?@5Y^&mG^8CwA;Zb@K&7w@H6%)zq)EDnM-;k)-EJt zPIv^5s(P^0DmYnn3(g~nOr(VvNCYg!1V1wHsKzE~x=SB1tr-k-;hq7RcCXgu z1R3}o()h1`I62`~>C_yF=LyQai7H!9jHA}4`i(?#~_4ubGBXgY{z zj23$cuv$T(|96T5D%3`UaT_p7huW|b_orUoAt7mxDEr%GOv7B*&6j?x=?<*gYu}H< zB0Jhw1itCUzSoZ*i)LTZnhxWtcEHd5O+rFR_P-JcE)d-rt$b)HHX|7?j7V&UAE&3A zE4IN3+shvgsl_CxwZL@v*oIdcqWS}4n4li7#PTdE$=5N#te_mOU%q$MK==DIqM10= zEND_`e z1;;RlaIUIm-Sah(OT$m#T`-LBVPdF&)GbO40z8UQJy8}|*JKliwo};!e@?P4nPX@X ze0y@RP@I2yijJiKfuGs#^In7%Ojzm_kFpzpaiq}%oeC+=?nfZpItD6jQKBJ-iGfDK zJ?;H`A9dAWooplSB)ma3iK7^W{#r^n$AcIY(*~a-bQSDs4VB(hX2Cj+dKK#l2b!%e zq5?y#7ZWoQU4vyfsE$pGMh1}9Z82CunYp!ZF9k!+NDHDlDqb&g9^TA6El>LzeOfR1 zbIK9OViT^5_#YP<=UP2I8(E@f!W#168oM~t)4g_!>Uazg8(I2ZwI;KiXN8@H^^@V> zxHjh@Bqe=hXw=lkb9Nv-PiYziI@A?{3q(umb82@=QoaVlIFvH({c^oT)-N8bYjLY3;atv`P z+=3}nL*9{c*CL!n=VC<1e!nrq9H&eM*idZ9(l4&IxGQ`5LWu7h9fS0+zdOk{Td)>n zZ)0Ge%*5Ow8RP{|6zR-c6BvD4ypc@dx4yHuEmen}zz(jT-G4^OWU5DsiW*i+;k{|{ zsIJe(cgeZQu>AMyXs@Machv4}yb#8db`6J%{v7k&O}4*buzu~Fh2(7c@0)5Dajm=B z=L1dz$AvEnd6PfG3MJAA4|?9Q1Nj4-+$E#=qV2g1R-lP|*wW8Yt8jHREOBk&)zx%P zKz9}Cx+#V{Re^HY3c0=>SZ0r&TJa|!_o)6NL)?;!opBhC>WOuo-SR%FxRZH^l{TTU zX;ANc;PmO+ZqLWbg%(`qZev@=daz*gxulZQ!y!h%pqJrLm1rDKd|l>6xJN2yc6E3t z+`XzD1*&JEx*kp8PKuQDKNZveh?B7n2M+UZ9ZOU*EcQR90IqD2_62#_L!W?E|43Sss_TAn%IRrKPPLxmzJg(l*FA*zY1hK zW^&@bt2oPzs%qKp78-zXD&_Ajs6vnZPYwANZ_WFT0+OyVZw15|xNDAqoT(HUoKQ%J06HTwp!tB`> zA>vbrL`TW5zRu#)d33>;G}9=dQf+7#A$sE{_qBJl%6+Iw1=%7O#G!A`C$Cm-6XP=Z zwfjD<<8x9^9S@T+I=c|{x4?uWs9WI1@X4&&;*gSXX$I!yW3Fr@(-fO6yHkb2msu&Z zYwfs$OC@pxnP@P?jCN|t3uF3;rbt6|*f zmz-!?r>mYZjD(N$)RV2*`}nxn3*03e-()~%NJym0)5vxAa1~u@qo|aWcCt-+Hf((! zHiBoXr?z&}%{1i#pAAix1khDE5rPGsDa;iWrCG?(gsc;A% z9+!&ZE87=Z;5iU2TQf#PfVP;Ztj)XV8M{r-Wh!o2TavKeHO|v2ofUrw^%Wy%JB#Nk zp@w=oAcFfP^1{dH*_VX5tANfTyLp16!i!>xRYQr@=iAHU83js(pBZvMi1ES#CZEI> zo>EJkgImB3Qc-})wWJzYQvW0tx&@LqZ7+?^<5qv24wi{NgX9>&<=j!oyzKHA7Z&*X z8Z$SwYeE!>9y}i&_yYR8t%W|2nrnb@!usZXJaXujqNr~-C$oM>9bHHFcbQkxJqLFf zkz}=hKm5HkO3>=iZ^&nH_;Zs<7#-PjeOC0}qV`rh4uba?xZJL^9XO^?V>$T@lniGx zJ6!iuQf)=Bm2Sf1JH&3+oxk=%66vvVleMMw812vNgzleP`y8=mImCYCxxi<;F zXxmgzBf61I0B$xXb92o)c!t|H-#UA^PDyA{^U6ww??IBjekE(@u9{ajXDrEx+FF&q z4Mj*`ICy9K_a>3(;jmfk`W00s{nMBrO0Ut_PqI=vzYG~6gh z6Y?J$HYE_DCI4e^ZJZr|n4R!jl^8Vy_ZE6i$Qc3j1Y4qTDWx$H>!|bA`&_~fIe$C_ z$f31L^GlP@g2rhHn?_=RwFWRM30_}A$`(P9GVHKpM!#XV6Gxs{T@I9MX9b!JU$mt< z@A--`$=yjLGy0t2tb!#~i$$xMVraB;MXt-!1Gi3+B9tW=QMO3`wO3e7m7LozAIcGq zi1QNo?>HZL6c;iKWuRMWB#Fg_q;n5|B_EUhXw|(u9maw*#B8-w(rD)l6i0`~z~H2B zu+M;C9iM5QcpFI3zcyIk&c0zm^xx2x?tiuAS6T3CGFGRfSx(MC>cyVz*g-tj)wxgA z$B0Ks+p%$y*p;^Xae>4okNR9}mYrDU{d@48&?v?I#h*U@@yh+S!aM~}Ye)W4auJhI z_8c>Z*?}dd;qLCi4c3~n&VOcL*?ka&;qzgEt)~IR=d5prv%hx@6rit@l{C46<$dF!#EITg1(0@1pu*eQ?7NK{Q*eohJ@ ze#RvK&yQfnJ%VE=b-R9u+O5XYs(m&9?*I&GK7iu9p<6}OSr`L5COC{eSDWcgk6n9- ztc2=Y?Fm|kdD^0M(6o2rJ^TxIjsfL(B@#lQ?QVl#SCIGWvCJ}hu6CrU{Kp~&+6x!S z1TTMJk8}zfJ?HNVX3&z@mg%MZXLz9w@6AR#DAcq%4`VjF1Qs5XGjf-81j znf#SqO>`ozoxFxTd_D|g!}PH&8IIMx1YL`G+it?PRV~6Yah$-5{wCWlm^tE2SH4jh zLtc9$^dj!wtpC?qNoVxoEqgW8N=+^eP1MzgibObA8?3Yieru_;rdowI=5v{!%_%qK zGkTPfAEPP#8#D+*C8w1@{GnLCWahuj!u!;J3{yW*k;S(n(#m#ET3qYua`b{c1MT=W zstG%PCDIm+5i-+nf`tD2;%;oQi6(*Z2+X*>OKvt{H4gFCW7CH3y3e3){pHXhlAGE#6M zBmfuHn%IU#bFq;)DL&A2CUWQOAP(&I5%`Bp&|3e9X@&Ac*W~OD@hCs^Ar4IyYZl|r ztP@FgeEuPfEah92|1Z4$SO>WlTdmCla$cv5lMHUWeY9%=^`um^gkui-;ny<|JHbn2(ee91e=6A-|XVsqiL zAZejksBrknPt2XwHJnHEQ@gj&eX#c#E`_B^U*ND?n5VXgXS%)Q*@0Mc`2UG5SL5Q- zTYGE8Hkh5yq9^QZ;Emk&=Y)29V32^rnvHkVuwoxx6pEm7&(?gkz1Y`;X@q3B6I4C=OM0Yw(srnn+lpNe` z^+KaCzU$)P)GffT_7o+GVY7o16yB98UJLVCNcZ2{gIukKMB=~a)6i(+Rm`&g2xM@^ z1x<}XNNy!rQ+fYDy~_At=?IQlWil2K8oz9{e>t*dDG}5UnHa0LK#)5aMx$po@M~A; zpMxQ$`YH@N?wfRZ0BsE{3a$wI?-vY>5j|*$35j5Hg-Ka|`Ywej#ugAfR^we!D2X4Z z$BvgUX5|IFEqt{o>`oeWdb6@-9z{m{M5RknbpRmJeN@#JVR zt!P+eLZQ@IByPG$Sm*zmPt&H8KgQxQxjSM1!T<3?g7i5u3b}3~j0yi*Y?TjRB^urUu_rY->HJWMdOiHhi~v(C>CR@`fK7nO|X_em+gV@3Nsh^js8}k zrEm^2Ro`MUaXhCnjm|*pqWvD_ONWl2|wuW=*YW=S&W3 zNV5O7#!P&U_fPLw718t+?%(%i5NBk$GUA%7#Tecn|E;9@nyTaaFpz1o9dY^K<-cX3 zZNv`cZ^Y})?;N{xU{ zrdo?6Wtq7NM|??Ge&;1mRp-2F)-m1IG8!f^-C%oC^C?c`*-npu!zC!bHwBXRI#`T{ z!eS!ElM*8<3D-id{Wrp8*_vWl!hp`R0(QLLa+9iQON%I6lYaU4hv5xd!0 zcBLG4uy*LCAu;SaUNw%2a~QVyV4i{+Y&?TE?S{P8_(y+TgSe%453M`pn zgjAslz8BCEAb8JC`8+7%yCGoa+#w9J7WUKmub}vGlB`G60<^Tx<7?ds2k)gC!H4@=y~emahn%ZRoR_!#k^Z;k+y-5nyJJ zy*71j8>}dB4-Ox;vw`#EFpcwvz8`vd3M8!BY$e+CR#8@mU^S^#CV4Dp-!DumV7N6V zYfJe}a-;A-iVHlatEct$f)kydj`QK1&BcJZa`)zT^QKpKA=5k8e0JmE{>Een_5|Ln z0cbW5I-3U6*2TDjf|=KmJO@iMroEz}q5J8FZ^dDSVC>L>P;VaG)N3`+-%9*l{i^=! z(vZfSo8V!%2kOZ57W1m%Dkllm3PF-G0U;+-@ypEWV!%hW$MWeX0_EvXv z+6%;)DPu?;k9-f=>ccDg&Bs1qx#XruTJCL` z`o<5kNO-K@zo!$~3_mLTL-MaN&Th4s@UJocU6F(t*?TGvLjT?y@fd+z%eB^~ct_N6 z@IsdXHtQY0g0oQ)0d*gcV!hU$U$l6=-9fajk#(yYsnR4`oKV^LZRCO@P5eJ8=UOsOmA@KGU=?>eB-6e(7YTRz zR~%~#0NhyZMv3Ax$wi(%NSg1Icj} zHOH63zSaf;r{;WR9cOY(Nl`V3yv^s2fL>3LeHIOMUnBMt1f11E(NCKF<3sudo~jgQ zMY7U~y*_>~7J!B*gzb)h?by%2ffTup{!NbDuL}oR;F68R61aufDU zW6(9ra?+j^tzl~XRDt&+F5l4b)|c75jL1AqrL^Q*hAMdU2bd$)sgM8emDi)o?L*kz zV`t&0MfxIRbBIdt65`pXdj|8<9lv|qb}Xe16<~)H52&a@@VO#tKSB45_Zh%rogU;M zhzMSw<(3^gC9UP+bxD@_0hb9IWkG#q$Gbxf)ptrR-*DB&#XoK7(BX*R^ZaZ9$@R`=n zNEtcJ&Op#iv6K*tD4W7$&@Hre6eIX`fD8dL=$!tbo&hsZ@$)*Bq52rWw2Kt>j1Hes zU2lT<0)s_PUNbxX+wuLOuS&J@@)LH`aWD8lpeo{h;TOo2J+!9Ql*3Kh z)2JIo=ID}b7$6mkaFdWQ+HB<)dtv5fCX1;(0h>s|jwJ|(p)$P6;mW6F3}GN%V*aBe z)exwT|Bx`wydHTX5aXdA?`eC;QokQRAchfd7x%Jpg$ReUWuwYQkn5)Q zZQLXws8GaOYk&T|F0^UyPaj;xk(cu%23gdSj!rFl7YwX8h88ZYeZ*bnkEb!Sr>LCy zfv6^0Rf+%W_8D?sbmrp_U*Lu@Lr|q*s|l&!z&2#3A9Up2A~b>>tZtbQ1s+#SIL(g@6l%cNhJoU=F+0G5J(WF4G?xE?v6 zDpH%eth6wv)CynfYwQR^nKccUg_wcfpFWW`f`Q5tlrf&XuwA5x-%v@fEkJSH!V~L~ zuS^IDu-u*1Ho))LJdI?;O;Vq&=ljVu0~Pjx7XQQ44%>cl5PKD^VS{0?y@?A5dxJ<~ zL+BA{mKg8F!+MowEhM~d+Jf+%KnfVt6dvB<)1!r4MgnoUAH22q5B@=|f(hZ}N+$J? zoE!Vg!q0)1TWh6ULvoi@pC84;uQd@U%*_H&|*5M;Y{f>uc9`-hO^(MAtS*K#4S4AohRzTI1C#E3G z0awPwsY!0t8Q?QIMeA0Mh??oV#?f0K^uMAwNA?F!aJ31g?@Ez1P?p2t|EdZe`iC4M(f1P9{xsGXfi)uQpMvUzr|8W zM%R6Js7|w6=|7ypt#9vT!bqgYwq`k3n)g=`0%#G7##KgMtlvK@`bwbsvS)t0^-li@ zb68b0iku7czMRA|XEVgXaDtLhIV<7@ElHj&*um-qPPSG7( zh{Ft7boH|S>oQ0z?LF3}A3KDZwxQ|F+sFQ-Z>FhLZ z%W5=KyP8tYOT1p>IHo z9=bH|7Dd93>tBG|VthFh+j`>cY9g%Qm#0b3&BOSoPR{A*(56T%Q3m`^wCrQLQ4R4vZImTSt{XozU{M!>D+++4 zdIrfAHv2{_LpQ^;D&`s|W=D}k7=3>!=0veT82J_r3hu$c31MrhrA%VHgH;z{Y$F)~ zP_xSZvp;>5{hBQ~M1k!Rr z8-9~r@6GlXF4fsqt+`3ML}?{th-z5rE$B2G>!`I;mvZzRjWjVjGqP7m3dU}5cA z_uo3YA)uCGLJ~!8iyjz6~{(^F!wn-p<*=iD!RP$0<>M2J0& zgu|UDqH$lgv%CG--SJgD8CTm!Yqq&?8=c}DqkE0#c&{?q~CvLV1v(c_FL>*^RK&{e79kx?YHrJ9T z{iz=oJDajwjGZiyZIwdQHIc$P(XiiSnnFX59hZ4s{UfQiamk4+ZE|u#Y1+L{XWuk{z-vysFy|NzQLe3Bplz~* zsXJlqcM6*=NNHdz??BHH$w{iMuP%3P@|{HCA{g*MLN0H^R4K_R2A`ENkbE-MAz7?$ z5yvz7H8~9KRk&jGUEbT; z(bDRn{-~m(adyx3b;0-ap~5O%V%S44XJgxzY&}E!5dGD+MCAW8H}TY$oA3W?@>mz} zJ9B}o`c_~l_&fC!&>8S``u?=Ebocx6GS!t+-S>3%`(?0P?en!JsMu$r8IVB!`1Nl1 zV__nx>kW$SOtoCE~{Yk!Q z*x7Y*b|@lE9B*!jW;gw|i1pO?ZK~iVH+eKS+xSIevQi)FL%vcW)H27dgs9nRju-gW zl-esH_Sg7Jn|xuNn#0ja(&4*;y5->oqwIgAi#*ZD-25+wa(?l@{NjHJiDeMeTJyD9 z#s{>0Yblj(`c!kXS@G#jv9&2}itdj2V_<|X*3merL59`VT2V}8L9T`XZD#dZ+8$M#$wt>n zPb=L>{#$rv0b6Y+yZ(9DD&DgXmIhX6TRLnaPy6?8{H}10D1t}YXX(2Xxj4p#YYUVo zmQ@$}G&nMnNVky=;ivdA9br+D~h;WpaMi_$^3J3Sqx294|*M_E) zIpa@9F;<4qg6ZQ{=75Q+dwRVVM7SLVq9RCar(*ic?9LVzv>TN zAB-hXFO0nXs1jS#SZT(um0HMw<+*7!K3f8U37knlLV=y$3Za8^mB=`^1i|)O$FLUx zIt#`0Hxc?hL4xrQ!9a$<*K2^(?3Bqu^)2e*vHMG|ousaBX@KSL4h8+Ni`vv?c1~Cn zbdg8eh#=CLo{G%2r@9AC$-a*u1i?>dCI0C;|e1_CprcZoBakx${cM5vNe0_!U%YPOafm9m7%(tS=$p;y`7`LU&t4&GN+<1r{_ zX5ta)CP_-0UEWk5p!px+NR^*Ol?H5J(&+)qPs#kV;X_31$ar!1X>W)pPN@ARPPp#u zzS=ri@9JGlnlSv3M)*IL2)V@#>el)11<<&ig^iDXVqU@?i)w9xsV%;1{+TjT zL!rNovQQ#Eo@@4b8H}NKlxhx4{nVdAn&tz`&p!!9Hcb|n-PtE+cXf=6dj|5F?*har zQN%h_Lon_Pj%Np5vvMOktH((udEzm?9%ECQ$1;_8#O{^1z5q1^a1FGG5Xsu>VNTppAzs~`L78L+L$Iss=QW1hLa<)DQVz(DDr5pIO1DAkOYOL8~$1@vBx*%tIH*n2D%ol-p~~mK*vkyTv4EBTUc(WAAv@YT#>H1EYmut zgTKg+xRghpDb7$5kkfcRumHGL{m_s4 z5zzcMj1t)mACkV#3m{_%G$({=H$OBz{@zw4{f))C{lOxxcifC@ZC;x!_Y zeL46)p^!Y?#0OLP-nU9%TziKy?7l7?6TU z+Y;q!O+sHS2uuhU^bnM)IZ>R~;Jh@qay18pb<>;&Grx^_&8EFGb@Y70@zJ!@clPho z2RaNq8Wt!=%NhSRMaMG2zVUiSI41E5%@_LT`kAn6<$XNkBXDN3IJX^de9Q0iC8fQ~ zLkNoFJE85yY+AT$anXc_>=SyJ@(f(B&Z2$A(NeGpz(FARfY#{EaHJ1$wEmt8tw#~z zOe*uY)$96wLJ$H72Y^#yIXOMhA)nUtTYY2d&x}(2E+LOw?EjA)IDce9aR2~L$G0RM zkA>wZM}(D)hc${y<(t`x%tHsga&$QA+lp3c9k02K<;X~A%x&hFjn#8)7KUYOBg|fN zY_4HB_Wd*7FMZk&;%LrvFs%BX#|6!Kwu_O+F2@X*WlB@*{Rlk3pngMib?953-n>_W z^$Dy-HAX^6x^DQiR%ZSkRAe(1Y`GnjDWR|LURlQok#0%w=3PdIB#R+v_e+e)E=e>E zXm9xe#7DrfFUsAr!7nSQ=Al}@$#kg4PFI(b#(W$~tQ=;=ZStqfw8M>JfZEQeH*f<_97%ea2b)byxKn8(p$I3^&7L)EdG=#koF1Hwg?8mUYTv69 z3rRg&8*kbfbISFiSCsvScs59{2bk39?BoY=3oH%`eiDX<9&NfZx8T6& z5`C0oxKDG{TqfM(e;}>h&NUA_$CjiGTIwxU^*A8rK~rS}%*#hrt4%3SzE9IubfIIm zk4O)G@HmmHdsKBJX3mF9Ew*b{Ur+oK4VgU|u8^D^I2h{qIbK*s)6auj(*DFW^*eawi6cP*i9!{{%Ijgh@S4tjfBq=^xPXw%(d5iAD3Ga^r;uA(6?V~ z9o65dw4$Fh1~#slo=@0aYLA|&#~!{pW4edYM=&tHJBpyO5~(cpn~_a_rXdc(cCiz$ z?-N|52!CKyTYGq4-%7!oSBNxBda|x<3;rkx7=(=|$=9@pyc-^VJh#-yKYY!li7_3M ze=1D##Oz%arTvSJnTWAJ!YBy1E2U8?s|1IZ5_1-c1FAM*HwWG^nD9H-J)^8*$(Q$c zK190qPl@!Vdvy<0?Q?mmSJ=ooM42!zDye`u*MO2*es3lP`*qy-Pu;8N)oeUhPVhOW zU>xytuPN}2u_I5@$rpO9yN)(o;njv6Ld7AU-u=v6{Q20oldr@9Kb`seDM{8JC8{|4MpMNQ6r?o6k1=oWKRj?hL&d79(f}y8(1r;;^&|ThMIF0umOm&P}4Fa+H=O*`1F07L1c5nO*bb8THMD3Fuz zPnnHa@NR;04|Y4n2;!Q*}BO5?U>CGE0K8Q1!neTL{^j-hb!!7a>q-IS*# z;=s7(@u35)38#}DNuGFj0s7DTl54%tWS$O;g-M9=2(c;}d#@x6vKbAYJw#>7a`~X#v7IgYqniyD#G9s|3D>X2?ehxPP65 z_NVKzwvr+YDI*Mv9Tpy|jY5a9zEmgKvbnayFf<%_HKLg(YU>B|+kedV(%s0KVM~7- zUfyv(}og!1?-5hm-ueuzcHyJqnaLmn^nAx2&SXy}^c$dd7DZD8`w;}C!u40c;UUQ}RIw1_Q>MYWuYH+$4ixPR zKsxTJWHZk5{Qa3;&&dHbEQ4bN#2dwC9Zx%i@T`}zgX(#+C6yt7J8NI`v?u!pKI3hJ z_@jm<^)I)qxO*wm=2*{v#{&kl6lXVYN-HB8Uvwq6F>1nG-#-owEu5Tw=a>TGC zX_e*<^i0}ur~dblwKjDic2p5*Di67ehierspq=@-#^jBV*d@>9H%rsWgSrnRkiI_YQOyQ V%P;?q{|5j7|NpJ5fKmYX1OUb6n@<1$ literal 0 HcmV?d00001 diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/Chart.yaml b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/Chart.yaml new file mode 100644 index 0000000000..5b302cc1df --- /dev/null +++ b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/release-name: rancher-monitoring-crd +apiVersion: v2 +description: Installs the CRDs for rancher-monitoring. +name: rancher-monitoring-crd +type: application +version: 103.2.2+up57.0.3 diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/README.md b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/README.md new file mode 100644 index 0000000000..e0b63e0268 --- /dev/null +++ b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/README.md @@ -0,0 +1,24 @@ +# rancher-monitoring-crd +A Rancher chart that installs the CRDs used by rancher-monitoring. + +## How does this chart work? + +This chart marshalls all of the CRD files placed in the `crd-manifest` directory into a ConfigMap that is installed onto a cluster alongside relevant RBAC (ServiceAccount, ClusterRoleBinding, ClusterRole, and PodSecurityPolicy). + +Once the relevant dependent resourcees are installed / upgraded / rolled back, this chart executes a post-install / post-upgrade / post-rollback Job that: +- Patches any existing versions of the CRDs contained within the `crd-manifest` on the cluster to set `spec.preserveUnknownFields=false`; this step is required since, based on [Kubernetes docs](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#field-pruning) and a [known workaround](https://github.com/kubernetes-sigs/controller-tools/issues/476#issuecomment-691519936), such CRDs cannot be upgraded normally from `apiextensions.k8s.io/v1beta1` to `apiextensions.k8s.io/v1`. +- Runs a `kubectl apply` on the CRDs that are contained within the crd-manifest ConfigMap to upgrade CRDs in the cluster + +On an uninstall, this chart executes a separate post-delete Job that: +- Patches any existing versions of the CRDs contained within `crd-manifest` on the cluster to set `metadata.finalizers=[]` +- Runs a `kubectl delete` on the CRDs that are contained within the crd-manifest ConfigMap to clean up the CRDs from the cluster + +Note: If the relevant CRDs already existed in the cluster at the time of install, this chart will absorb ownership of the lifecycle of those CRDs; therefore, on a `helm uninstall`, those CRDs will also be removed from the cluster alongside this chart. + +## Why can't we just place the CRDs in the templates/ directory of the main chart? + +In Helm today, you cannot declare a CRD and declare a resource of that CRD's kind in templates/ without encountering a failure on render. + +## [Helm 3] Why can't we just place the CRDs in the crds/ directory of the main chart? + +The Helm 3 `crds/` directory only supports the installation of CRDs, but does not support the upgrade and removal of CRDs, unlike what this chart facilitiates. \ No newline at end of file diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/files/crd-manifest.tgz b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/files/crd-manifest.tgz new file mode 100644 index 0000000000000000000000000000000000000000..fe5d5c8a0bb6fcc376d198b269053bdfe4baa185 GIT binary patch literal 308610 zcmb@tb#&ZX)9z_zcFasMGcz+YGdpHx$IOnInVFfH8DnN2Da>MoVk+WV>c)!qb=P(XhzpBG)<-1eKB7&c~J(A>XjPB2#_U!>`Ho;j^`Ub)Y| zj_Gt-lY4ZevxP`V2+5+*0F4)9XWp;p1mlQGkphp-4^Cga6W#tiI2yLuu5)7=(TOMs zD%oRy3)lxN+js;SpMH-;AD_38^pl_PzmHxpdfx6Tj%Rv)?M)o_aQc1>-D2p!L-@3@ zM_?;Jhrigx{BRh<)Zf`U4jNoI1;S@Xdj1p#iGEc7#M+IrPumBXfs)r?EMjKV+vDk& zVSi}%doewJ<@dWiyg%0WozwfonCbC!_>J=*`1O`V|5JWWXIv>07tg_)JzJ5p=4;aI zH+o5;5$K5@Qqzva$Au^)YA7*e`&m8OxIg@!jl8`@w)Q(GC03XLHzCeRU22kv-r&&B zVJPHtSkj&KJZ8c23YPNV)emku1l>b6;9-wuz1plax8OIkAQXpCJVerjd-m6 z#dohDu4282Xe9Tl=q0@y7SGT@*v7Lod%c z`c1yk3Q{8dmXwe4)(D7L#8h%%@lFABXD{D)y~7swW>4eM8?af0oLtStNEdG^Xhczs zl7AG2hM_xg<;535VNg<~7Ai|Akzy&Wr($pj8&Q&u4YdwyIyPC?>nVapE6@JAT-!ld z{57Rap}M|}Fg8spd6`JWCh8~$zE^l465Ed(hHg}d-w$q-;1@3vGl#Yd6OCiq9|69L z;t*e}Dfp?ok8b6BR)Cc}Xq$;P+mGPo-w=U&W;u3?;*j4RP@eBE&rdvj0>2M7XEZKn z6~bCDpY;)R=%K36#xoC@CPy{a${{rtU?LtcEtX2|L?uZ~Ef-@iDW3iyrlx?g2=yZ*$1K1@D2@PspeTqU2waT~1?D%eJiPbS@IkJxL&2vkgK4J-_ z6uMw2(Y_%}dD<~L;fv*%1<@mad{n%V&#NfwSeeTCM@g+`lZ|SzM!a*> zH0%S)F{xKL`05)-X&O2SYn`F=?7`7FVut3o-+c~?*^g@`km_WT7a(W%a;QFRlW|06 zHsh>bR{cSfw*gF7*;e^gFOkE{Vt!ib4P~MWyTkk6Qnl$x)J&vJ`}31hj>XK(YYVcfO?-PaBnbuKqiL9E_dYC4~rGG!y^%iK3T zMm3|bTO5W>|N36IcTf>H0TX~J&bj|MJ?m7W3f@j6uD+9l?h}(q6(+%GDz$bhdX|&e zMmB&;ULhWo^%XQ*^E&;S-Jdl zqt0HIfrO}BZmQ!DI5F88+5-p+D1qq?no+S7j6OTWk790L4Ds3RttG+78Zu-$d zXKWC?J-XFXkEU16t{H1{T{1#t{NM{dw?#Ghd~&f5BJ$!IFWGKlkr1pzI;yPt@dh?x z%`K*8O)!ZKY=>9aXXB_}=&KpFx^sDq zFmy>Z=yt-e^X>XF$P%okb;&mTp`b|%#nOeaB5}1~xhmlLD+xr*;|gMz#VU9@Amy0t zh0byL+2d|QLXy0xg@W9XsW_#KS^8}x6ci^c-dF0!2%mt++ z@1S@28$_5MKLcnd>6PMB2|k{@AAS#Q=kUJ29-J;Dy!#!mn$P%tOtRkyL=rtXQz~LR zwI;e>DF8h=G+K35*hlSFuKgw(2g9ESJwwDe8NzU?u1xgqC9@?|>Ze3$iE~`hibSQW zJa26Onl~^UQq=4KmHlz~>t*}XmFM++JLmoP!_an&c+ThhYC}Q~-jD9ap7)F6l27;B zp|RUKx5&Iyn~==Tbt}5+jSO41Nf7p95+E{9XQyUkED!O~s`*P|ggeBiq$kbhp3M;l z+x)APq=-TV$uhCc*u+W!b=A1NBDIBU>aXzSf#w4N)AX}j48g|-IC8A*9CcsI_l`5s z_$MO5a;_z`kZv0?)pdUJ96p%)n-1j^xm$v(>@4&Ir7B9vPG0@4@a*nlU8;GiqyN_V zo}r{n(CJ9;EA#nw|LO7iI2QhL3M&6Ze@L`PO5`lqp4_&;tis<@rG7k5T)Gr!7Ta}N z&gdF>+v7G2uy(fXY73d~d})SHg~jyidXT4P|N4(b}sF{Y!^<=-~c4 zlfnhqdt#-MTnC{{M?cy7)tyU}T=@e;yge}@SUZ_CYOa(95hHDsmZ5pC6Tk7R(~|Kpu&E!VI9xN_%ALL6M-0!-GkH_5T#$VGD$_CxW>ZbMI5h$*Cb4Qj!}{! z^`Td6?8fG6#%yT((>oPC5gBj$n7eicfsRKTzpZ~NDCA9!f2uN6&&51M}E=LniT-LO_B!q z!MH62@tm_y@X3Y8js`dK6Bb#U5KXet#_`$2cp9{QF{Rr)IqS!aZ75ZicTbMJfIYMH z)6;W#25rH01_nbA=VK5}r-157Jp)|DGDOd?oC*}K=cQ4o9u!P$c^K$JLcHnQIV@|K zPJ$}7Wr~r*(Q%_O$8@;PvgRT~c4VGM`jr(7wlR0w0T*!|2zAG4j`tQZ>^4rhjO%=W zczNPiGeHT{G;<|r-FlU?PItzmP1lnVUuO}Ql05qL2Yfx70Cuk%Lt&?^a_QAyZyr~g zqud15?96^SmFMC_ZWM`JEvw-zr5+7{9mc&nIHAeQ^5T5by;=1fygChJv9 z6K9ba);QxA)CS5=5HgKeWLhvvuq0=};gb=N_g^BgQv^9+MultcqL|_3G*~HD$*E$l zV#TfF2jTrv4CKE@$PL{Kro{${|`Y^EyawT`9x8_9=}e9 z^J)-fT8Qp?#_v`?{y>3s7;)dxw1$nvdA^e+$uTdA0K4LxXsDdur9i@ zI|D04&6R@#E=2`S^cHZ*M+pke>1EZs(Axr~cP;hD0Hue}!v*P$Bw(kHVH9elbe^)6 zG)%UL?2RUOHRD4z`awlyohqZh%iutBEOuqEO~e=z#(>*TM9#zX1tW}+3pX?$T)9?Z z&Da9Lh9Fz`cG6ng#Y7zB1Ou7R3+gOyrx!m$zktIK72^PQWw{xJ$pPf6PHPZ!GDV&v zfj^U}=I|Arq*U8lVjRj|#?w0ddv~2RQQlV)@C#ag;RI-A-;WR8Jp@0CRDSEFR1^++ zV)5TsDqSk#{5Z%Iv`eJI%ESq|9Jxv1({tvwE|5`-Oc5oNyxY@}M|xYqhr=99^AGTJ zD^sn@mH4$*mpV6$w-V2JMg_Zwtigj%d#oD6H^^Gj$rX z1OxNo=i&1qodYNePg10wEc^AMlpBluBbxOW+=bDCPmmt_qHy1-8Z5qLpxKzVjotEmGctp%Xme>xo(<7ZcJxDY!#oPPY@V&#z3X= zGQ6Q(gs0fDe+wipMfJtY zCbWNhRsy&E6dyF`yCgv9z2H|eL$v+0EywR%&k0Hz)>Ut}VNx5oh=p4{@_0w<;I3sN zlZ_%{ePwU~>D_^XVBk;#VzNNRPlf{tEJ&eZ;J`-?1P>&^{l4l~IEfDz12ELyCkQJT zACXX@t>s%&fTEpBV6Jk4V%uzLvj8`zL)Y&craPfszriJ0&fPGkXAP}?Mmg2Yf=37s zGoMQcZv{mNPX+jsQXx!1=ImNJ+<#|v5m@AJd%hgWjJA79x*I8dN!gSSZs>({A_mJI zdQek|(u(h#O#J1%ISQ8y>i?DI@n)rW7e=WtMWohxcRl%%5AErKu{v^2(hR1Y9yz;K z>P)T~rr@#wN2u%*SCXn>b1HZE$v<5+-RQCl@gf(MX^e5InX>Css;f1$|E)$13_fLZ z1#zq8n4$XMw~oF*GX9W-aj3#u`lFEu#tsyZIn1G`0$$J0agu77oY0a-#F`$@Fho;z zo4DO^BK)Wz0h&l3506)~hocMWnas5g`PnZ+puO_` zIUxK#q`x0Jasx_m#M`UI+x45Y96~Mh^b3PXGK(|A2p1NmgDtC37TT{fBF?8Qw^^Pt zzRf#ICyN)Gd52r7pSQ4eY9mcsN`dtc)embp*5_8DtfD{@XHji7_V8F(>Y0&}2g7Mp z{R#=#jz4_%46A5vlD8YXn_aB;3o%iBoxJB5@yMO<&XO2u4m=UOT}jCARPwaV7dJfO zbj3vu3T|f=#!V+-mdH75Tc56ViZ{h?`GmkJ(@l=oB_Nb@S( zf$;Ea^S#&dg5K=ML0I7VqZ5dr&>F zioHUg4-gect)@YPFc6hH5E-H(`1VorI-hO(P^Ic1d`s!N&Fhv{@7+ZkH)`YMIi$t< zF|fsY3p9b`dUtAuMP18mzlZKkGfXB1Rz|eZ-ZERs6hkkuZifxYDuU4}cWXr=tfW0% zb6eHJ(V13PyRSZUmUK&?D7AXK8>ZiHP%iwQUsm3oksWf$<+1B>b{73<1`ByZE2FO? z2PM2JPgKZAS;@JPeI3<5rc8gT!nDaU{sIIY9;Nab*X_>Z(M)r|zZ(=Kuytq31XObW z6LKU}qBsl2#)qEBYs`xHa01cODFsKMt)lFL7pJL_1P|&^EyFYIh~+L>cWxU>QPpd*~pwd{b*>JDP#kn7^BIPapK$aj9zDsIgHyA$0bW+_J*o4n|b*SrnuIH4=xu}bgo_nf`^qmX% zY>@Cf7qY+@qzZX?E_E)#vU{DE2V!Syw=qm7raR-!9uSb;?^X|oDCB(Eyl<}35mkG` zz^00maX`gtkB;S(!rsDNX=?>n4eGoU~L z$AJw0XVo9fRMoVyWHKC{+3peF*xk0_LQ@7g7Ci{FIkv!fw-(8z%8<={!www;*4^HJ zH&bmx7}D0X1>BE@ZIqk-g~h`@TMjn|6z}@6xfk(+N_BUO-~Z5qPIIe zmDSa3M``X$W_!OAo{sa(52$*jhbbsxsGzM0K9q+hJyqqXJ@}2ZK#SpSq^8j8*Ibh| z{oY+pd4gyEsywk+n-)bwg~yq}cnf>{1;Lj^0hVBix^d!0Vr=#85Fql#@4wZ|(w-lQ z5DTd+hCwU(Y-5`gkzIM^-B;!Uzf0BOh+Nfc(X2Ha(SWq@x4wslL&z&Y{pR8k+6$SheL&Dq5$*7V}gQ< z(JSxWaTkF@hl{ZXGW(2U@?zll^K>Cb*i^sIWm5qc&@MCO6W>6rc9-|&5y__8SaJzq ze<>6c#9l-az*YewBWMg1cP}z*6m8n=WewY_e=D;`|Fy%*P2v}uJQ~LvxQ#?~6P23W z7k^+HQdIJwX%T8b#XK{7Rw&ZbqOElGCoijF*DSAqsP|gUVMeV60)zhGRb%_9G%KFj z#n6XOKFi;kx+Uxrz)*+FxUg%)QH?K{`!fQqV|R~DX7Y9DTP@O0xy;uwT-IOm*TUST zLj2B0THm($ETHpX`$w9f70@2^(F8emWuSr@eA8XnD$8!F6iihfLTEwedAq!_lZ?b* zq?#!kKC8d@sa;pEtX}saD^qXhAy6h?XXEWPsRu1)R##GAE@sM|!!)MiQJzv9D1H+V z&uN!@9sAw=%x%~YL!SNwRrV5p$SD?(m})Gk)J5s+5!MA{)9Xazz!cs}{bOiev4k!27)xw7(N4R-~Un_}ZWKi*zG<2Veq>N9N#9Ed? zETLH99dr`zeUa7+iE?uHV50V}mSb~n*^f4P>gtxRya6uKesL^kdn z{U@iin)vDU&oi-=yu%~-C%XFN^Hr`$yc8c_l@mdwXJj8JCV56;NCIjV5G}&4 zolGE3J*egrjyr_KVI)nXUe&jA=JRK${8~#V{HWy2z#_~E-JH@V6~`TAU<+o5sgTIi zmXqBE?{eYQ=ml_SECLUKh?|k>g2OJ61P0l@qgVuj+tjiFtr!rRnZ0l9!ILMxC9 zWLYSJ*wp62qu&_q4E06|Z~aQTF|r730$53d->IBXKow zy(gm_^7<*1O@l}!pbr=Vax@p-UE_ahXpnzdkA(oYRQT%y7!&Yx4g%Xw7r!WXXkoM` z9dwCcO_fo)L}vT>M#Xa;al{2dHuc;gv>Gr)RC8raE$^vbUkAm;D`ycR3r+_6fSZ~K zGWlRAKX(XVOeTMwExPrNLIcz?vI+7dF+xxF?uDIihw|tg>~9a{Acn&!G~-b4l{+?? z3^lu~{aE&uAd8l0Ojm|R)_Jw)@X(^Q4p>b;76y^%)J*5+I)o;+aniSN^Kq1s!yOvS ze;f>Pj8qQ?AFg%K1j+?sI)7`Po>7Ad&Pu|=KtfdafWXGMjBK1G&ahE+){I|9KqQL% zqKbc4gd7ic+=m!asz;)0LVY4gjFNfkg!RxWaYR~Gk5tz(GylTaG__3DPRZAN9F49g zqEGb5)42co>)G1b^5;wFsT-xnPy%M!Cwbv zCq5ObaIQ{P9hC}(F#Voks0?KoRPZOUl& zlnfD_1a4oZ^B>?E&CjUzB;5r?mO9?{|_5#YLY z#L!+3x^-rms@YLZO)<^@rfd1d_nnE>%qk>Wt{W!{)l>5k)4sssjNy zNEejCUr%jz!U{AuMRj{&cA^~OPb2)(8pF}aW+z}gHGu&P0${BJ1 z=n>F+xh;*>`c44I2wVI(fEqp}(s*0nJIXw7Z2eA5(S*qUh~9U&weO*1fxSB?MKEwN zf6~w$C`f}R0?G!+zyd%wU~B)E_5U=(SO~Q5fIAT7(`ZMzJ?6|tmdEo3{l;pq8wyd5 z)Y`wqg(mIx%_1r132p|x#5N_zmJJ8no4@<@L3CtCHv!rZf>a2u93nj1+?bSVIv_Em zNzAyNuYIjOCXJFO2t#8Ye2#WBfURf&~hqf=`Y%k9frR~^%iBaRwN2QPw< z0&RRAIo}5C*NDxZSKo)K^w+Ig4Bmn@rS(X zKWSZF*#DqnPN*it*ryJwB;hYdKRvfM)WGN|T4f~b6DqQf(~g*@1Z^qQO*WCC22Z^5 zt(FlTaXJ=8jQ25|C-toy!$v)W;`bnA+PPPycw>Y^00KciTa2ci!c_J7)afxN%8iQXAmY^pUI?}?hN+ulavkTopyissdH*e0Lx z&MYI!#s;wu;e(ce$s(=hL;Nc+MN?WBc8($6$wu@LGD@IOGPyHIzwhayDMIQ zQR|oXvn5#w>R4G@uXk8EF^SNk=svQXEv#!woHM#JB~RyyWq1*3Cx_st`|E?L=SY%K$dK=W}udA!hjv%M1gCe6Wvx|!0Btt6I~6^KhW_{ zRv{9Ja-s{>1^W+Zyc@0w!VtdhM-#a2$If*#>Ze*e9Mu33yA>9aYLHK`^mhfOPAIuY@8psnv z=vwG{;s0{=0^~&Z1RS8W>jXT488@~6V~ETDKnItH0*AoBApl|1I}k5uU=NUbh!GN_ zS_Au_e{{KTPIgI{o#U?u5yV0v5g7321Ev>n2P6cIk2WM$6Wx~-KROb$_Q*YHDJ zeGZYr%_P66gLDwyL8Uw>hk9VBI1+onh5H%0g~^gJq9L9U@0u0BC|_&F_Pe#KAEKl$ z*ez^O3I1{?yCEHidRhkk^;;ucM6EE}<`t8Qm%q50wvx)o!aAATT(k<`f+b(kYEJvC zUrmbrtYA7x1t)GHanb8UfI>5{LI(Rce$-gzoJlGfM0HLf(|S7OX^LCFzq9MQZN{lR zQ-Ki5720+y4KsH>=t7mOGV|!FR85daRJ5HAEMdZ&N5R|Y2|-UDR-2&8;u*}ts6RgE zEJ+FYED7XNCvkkjdXQbZi)#I6@QXW#w8-)N`+!?vdL{1rEn>T5pZfm3elFu|nVJbx ze1ecsbwXQQYC{LeZWJ2xZepTtIu6=QHOW$pa=|wXWBb4z;eo`fQue(Ec^9$GDKM%@ z2{$HE!C%wsz>e{=@vmN_+TcnpS&Dnq0?$xLTG`pPo9&^H*8=iCUvpQ>Sj4A3gbCOX zZD-|hzq^90jd6W<4Z!XsjhE-1UaQHs9xLtQLG;iLb}J|FQekkNJ(oA(Xxl8F+fMRy zK-B%H1^}{_Ukiv=H9a}mshcd%1L4J7QbXff3)YOOmoaT9X2FY_Og!B2J0|#ZGju1| z7e|g+R(I(W!Ew1mG$gdI?@@B_@?`kEv zIiF*rg!E`qkubk{3*(ilBqhXaHlR(iAw^e>g{D21J@8RZPdP1;OYNL93!%5`=`pvT)+hO#^22-ua(RpwK^uvktSpYRE@Ez!9; z<3&^TKUM?q7v)>yjd04YBsQ(;~{URyPS3N}llf-~xB@ z+8Ls5*RT6#7q5r?Z>8R)_7_I@2Yb~$LHdUeI9Q}s^ODD7d`u^Vg8~zDWkJ%%{L`5(SNXO>~bguL-+52={Q6f958AvJ>*;)MdjcW_8 zA)o!VHGC7p*!~DKfOb5a1k1+tY-s_81O7%r+#jOAeB>jAcsihktUOG*3SXcuPFg6V z%}w_F`kH@*xbWa}EhRfL#Wx-Is#*cbl8%~*(yggLXCe>ZSgdEqquUaN&|8s8j~4b1 zk?PhWBVA?$-$%8|5_wBeJcduCH*VgSGcOEEIWyUKx!WL$peQz2!{MqQ8 z(G}jm{AG2j>9ZWeJ`R@7u}_q)b#y8J095Y)#)Z13Y9paRjQ5bOWRXIcGsMVfSS|Pe zT*BgtuZgO79S`~VF6L@xtD8wt*{*;7L%#p!>i7y%2(<)B#EQO&xK8iURTvx^b1OZ5 zem-jW#5>}v%yg5e4Wb$pxt1cuEiH2C0jZ)92lP)I2cyP8p$*1vU48!NJAAHs?#0Uqi;&yAwb8jS`)t_|DA0rg#W^ZIF(l`pq21d6*TPbK+Awrgq zF$bzn{P9pA!7~5N>=okR-QC_8yjjORB{8OcIjZK|IcF8|Aws)rxIbc zS&PA@BS(>2Uh|uY6_Nl$p15;^2Z1+(Uw)1U#3}NuPgqk=Om%g_PDI~cDP8Gqnt?Lr zeC?rRi+yl^%PszCCDD*oJr;8pEQ=k$`)^lIv zsq1|9RqQvr1+Rt9EVOLT>h~W-{Q{)BT&+P9@n6cQacqCPZR>U;9K6L%d__@%P(QW! z0iagKEv**g|1Z?axnx?oau2P;^Vqs3eS47^zUlXJS#P_y;2u19^47C(mX>we%n z&}%`VhtPmSdexmLKLC~5`1mJP2OzHEv%ZdZNsYqCdFQeJ zf7OaYs#+$IrVfcOLVq@Ol1)@46t**GTnr`O6fI9BHpg54lcuwR0n!5id6fxd@;_-= z*X8+`-a=%Vv+MG9`f8%vMasiS88LBH(yzG_$YMBDk@=o*Rj5(be>8OfHI%fEwTf0B zh2CPN?hdATGB);N8I+e!JS#_OXmtk(Pd2FPNp1=Y3a-w|yizCAb!z{w=AkHzsA>SwhLl z%hSR3)&Iv>D}Fe|U@HnqKrn6>=05Ic2HG#VDAeS)!s8c}dki-hQeGAq^?m7frIxqD zrPY>`7_;t0QSEm#@=Lzj#7sG=Q&jGCQX|NT=*PO42CVVF9C3xfi-C~ z9YP<#`$QKT@^?Tu06q_>7`}ol0WJvfC)tGl!+)$4L4wZ!?f8#-&z68wu>aWmQvM9m z`rdbmn4JqqV*kftCIQS+r~)(uBqlClAlr`N7$F0PXBkM~J|N#^;OGZVeHS0U=-W8~ zbOt!Me_0p)8>a%8D!2jQ4&o5k%A4X`LUpUO$Uqx3E>$a_?&ZqglbzIJTU7ao zcml>@AJM5EzJGt?bPBfc*Ne{adkkI! zse-)Yc~Nb7h{2a3sV&LKIMOYYVcgZjPKk!XT(lbD9>l49=`dq~pxT$q=LImLbV+`@ z`}JJ@*qogRy9289Av)JP1S8iQB5?)^h$4p(ip+c*DCVFZ|cnMvOwWR5UB@#B( zY%E~`px{!v!mcQchOsgq2kfCCb< zV{)7_KCkora-g0viPBzE0QK`$yP82w*QFq1tLNk0-I9wiKhPmiBB%laHqdw<1P{d_ z;SF>Cyu8TIeDx#v#JPY}ShV_G7Rz=*o`M9pINXKG?kx-xowtre&Jv-*b;v{Od@$2A zLrznknlYi06gwO}^qJCS9|i=Q3P1JrsXf$;3^~l2j3MqvGFjjLP6fy-qrqEFV+svO=&ZcGK zFWpw&vUr<2DvsM5j`8SDej0I80 zIH-r%viQ1J=zFM#@U?svTR}a2;ABjr&AeyK2p&Z)E7wC?N_4HkQP%I52iBr$hS?1J zSF6UC+6Ewt)DEVSl8XB+p_R~kW57Ben7 z9a6O?XLBC0*E;i1;t#h^-mwH{EqwxejcT=ZzM05+aLAY_Ab6e+{280j?p`;MGxB#m zBx2z2ddL;qa2N8)PYv?5@!` z-RbHR1{XSdh>51{7HvQ2%pfqN$CuvW-g^#MH2o;F`XR@(cgl~-v8vsKqR3M+CwCwq zAdGB5p#;kEMHBc}ci%I7x803T3x$6O@vMRgXl|YafWs;j{)@vZfMG`f{ogrk2r>2d z5MovV;W%Hw79X%?0Ktp417k=0WqcmoevW|dbzpM_AYpF&yZL}9%YN=p`J(mYKLc_w z{!r5sjQ6}eiGOj}KW?#n0eA!8uz3>n|Ja*V82qm}tOW(if6Zb4JPMG1${~MoSnLQ9 zK8tlb9k0l_Ch+o)k)T!-P^Zq^sFsM`{1STlnVK2>?CvML2Ld_z?d+cSdm=8sx2X%W|QYLQ&OynD2{k3&DwTy_G*$^lY8krnO0Z3wjD>U z)|C+^P$ZH!wxb>9PN>Q*zl9$-1kJ#=MOtRcVEN1|Q;F`nC+!XAJ=cnttc%Z3w3%6R3ebx>yLpM$Ac5#S=Y;=_eL$35Tp<@1-{1Cah7GAtNk2ZRshl5AG!GF`O-_q@L!C;t4PSLTV`rzHAFet( zX6Mt)WzjF~!2gbMRxP~w$O?o3+zQnXqyx4?{Js5L!42=Mx(0Mn_wR_SALviqm3zhY zcce2l^qyLnZTM<_Q1g-}W+GS-O=x@Fc2)Ssb0487n*IU$m=7V=x z@dEzKNjiXMdO&}Q+`07XTlf13~{-clihY1+bI?!6~@$-u3acqcm0N zMw5s)FtC16A-NsiK+qo?4dTxyxM^7) z4QpbN9wddd?q8fX2rxJ_S583lpZxk61Tfe`Dwn1y)S5yX-n1+fcwET~unKFQ$$qb2 zWEIdna{+Aab>XTb;Zfn)p@eo;90Gx=BOjH(SBCzSM9+=>ksf>9N`UnE>cWjHwfswZ zG|$ig*Q1cW^uZoidIm`>3aMYz1u`xB@1>`H@xPXyzpVdDdT3k@0y%wa0J$%RcLxKj zulzlgxM-3*)CvYXD~wm0{@a68zpke4Osci!3FE7w9E`Jfn0+#5kcLw?X@yceg8Wh9 zE$#%W2<*#CWFNVc7R8zTtV8ebIidJnn40HG`l(jB^k?9484{HjrV2K&ddaMLW0o;j zjO*%Z-GDT$YsU66ce938Uf3?ZXavfag6$rx0=M8~Z+L4cuZoCsx6P~rv;w#^+G~dj z=PJ`6`s~* z2V$dH3`J~mY40~wXZTi5L1l%=%y`uS!%KTmDyv4S)Ap*&GD8m)$$c=huH^@ET$H3v zG%c2eH=rJt;KS2z6rGil*tP`**|C%&$z5-dya)UkPtyoIXIe zM>tVtwZ!3Ei8W}cho|ktJzBY~N-Hc_> z6Km>FaxP6lx}?GHhE31SlLvJF1QsXzKt}RNUI9NBkOVF$0a4NkD7+6F5ccZ>FFd@@ zze@M;KDl{_|B8}M8bB;gltEb@P9Xt?fGtiMpl4@GZ~$LpdGHV6(*+7<%bkVT=-LVj zb$P~yu>tV>V75Q+sSwV8LL`enH9I{EQ0&*nf4!%|IcR^o=YtLK1>_%l@IL>2={|(- zzkW~uvi`64G*A8@usKUk#AbCN(kaErq>_ea1Hpw1RA?ggx=^4Rudk^kQ;)D)d@)bF z9(J-wd=COayZ+n7m<#mqLl&VoWa-tWj$v3;bYtPwuYn!)x1JGAGH$k=keG!^T|+&l zlF;Qgq2@Hz7*nR}Z2STOD7D)l+%Q#ecwMGh&m~0UBtR0Lw(LKjxF{LNtb`O}S|VVn zxFToV4`Rqtq%*20hEB>nXlHw1ScFU-PdCasHReGCt^bfrXM3MH)qoFUcx zvC*LWAd~imoU^2V`m@|4F0cB{9^tE$=kxuIm?{+?Z8<)~ay7qYD@eX*IVizU^WxLyS$H|%O=l30RUy-8Eu`iF zFjr-vxFS9w>cFba8V^sjFLb-n^J>E%HuXKQr26jmjv zEd5rkB#lqA44$VeoK`qf73(OhQyo`96O;YFDb1aq>pY+FPESN-=923l$+yF!%_N>v zz5_4d+X-$smrKq$f}8*F;p68=UzaV#N7yA|I_h}OHSFM$@4!+d($%saF;3@o%Uyuu@hTfIOT^+%PMOvH>EM#Yh5$n7- zgOVks218*7MYs<-arxt+5IjPVpT~&8R|&|k12_7gD}nERp|y~fzxj?ewbRevFPmp) z7s0kBp@FY6S8?N@sJgQvF4UAkB>~rDEt?ArN zbvS<}>`{cgRaZM3`rv#=7edKI*!m9LpyopPYqW{=@l;ri!{F4LuUanC#N~1i+}@5aEs&6105cy^{)z&)W`+nOC-PM6jvbfS-*(#&nYXu$57K zvWc(CLLN$ou&kS6Ez@PViy4WkIrSYvvnuS$rAr}z=|Xc?^mpWJgbO{;Q`8tFAGm*W&X6;O~Uz;1R6fG5G*mK8g>*xKS6IT$0H z#+t5kTNwLY_4hNSW-7)S8t?O&MeantxI*~rZ6iIsIm$IG@^uH>Y{HnF7>{F^sGzlg zcTkIc#dgi-C7bK$Qb%|BFjT&Xc5aH6_uQT@EQa3;GQph*d!Moc!p9|bnk?dwaxed6 z?uTcu#-w-1-u%QF0W5%-^V0X9HItt|^u6;wKYv%$_`3ZDOsMzx7`pEMefDxs-~HHayxfl7j70ZDK_=<=oe;6)GhdxkDLTg7QzlEo5JU3TyIHJ@!;*|PYdZJ zLLS3l(%{92h4PoXQBDFbzZMSs&#R$dzrrx})*mM+c25XGZLZ@Xkcs>u3(NCd0HKE7uq;WPmv`?Ek<1{E`BjR@R z4aCmt;56V98)!(pSxAO7Znq>w8xrlKH+(!Dczi!zerfb5ac8aTmYthVv3MIrD=p${ z9PjWedaKSAY`T+|z7gRB*E$UQ(zHgX7JAnMZP>q=lrz75c8YwCMs%KTb<_+s|p}iDWj#=R67)X4~{lS z_0cUe#)D`5q||WmPg>$rW9Dv_U++mbed&JefZyG|oys3452}za7NHm`Z7hiT>hk#V zI8*(;>it~*E`4vvY9{`%lLD9q=6KS9G9eZ9jeXIS9hyTI!P^g2zaOSSV9!pU|NV1# zZSQ4(V78A;QX_6Qst|JK!h-0yodep#2AeR7 z@{3hlwbnZF)WT`?+)D;i*Cv@&)}$0!v^p_iyaMX1JrZoeY4BYo`nk&7&uWJiy?8k3 z`Z+woh7u5Mf`vTSl_QNT$xEP`2nfFSuoLS-zJhtl+W*Y3?+^Q*X4n(N0%q8sxn^*j z%$~8sVbdZ)s0l!W8=q#VS20&*TbRS?b+@yDmq^ zp~L#gfD7E&q>oRH(k#K0gs$?3eY>q)u(90cF})#m5yE<451ZN zCeieBAK-Cn7u@ztI}ltMTX-eU3`&pX)KeJVyoPL-?=aqA*2J@IT z$(EQeK96#rv)@9_{9cXV2A=OUcZGoAB>tr3;a26m8!sHbYK$Xid5KvWRhvfBlaK{N zhB|91u2x7Q#&Rynoefu!T0=_C3wo9Cs1lw2Y{(Nz*2p~xd3UUYM)SwWB*6!DQ$bq7 zL#5KakE-7 z){8%}iO?v_g{qIdfymI3j#r_0mQFfz7CLMC>q(YL@~(6Y(RP;xjf#eKrkDd!qYy&r_J-nA^4r^6}I#z^O` zJGG)8_`f;J4>fplLMl*wJzK_dLItwC&ivTPZV@olX8AH~+0(zV0UdwCV*nAmDse;{ zgheow1TI#zS#W~MyOfbQ4w{F!8kW@MPYzmAT58qJD1b?&@ow`TCs-*pGyw*iPxQbi zZyD$k&K}v#`)n4P2Fo4MgPKi1OamreI(-K7lM-TWeMY8@K8^3Fe*QHCp^c`NO$CGh z8)Dm-6^K3Y5a<^OyDe!{VkuXIYWYBouIK=U{NdN;ba-s))7dXKY!Iq9HX~d(QUwQt zVU(6NL6j=X5U~Bvc{NpOA{FiL`EA{1M--TpR)bH^p8Qj?kIDO8l~V!@vylBXDO{pX z*ve9Z^-1YhG#A${|BI}50FEu_`hH`d*tTukwr$(ClM~yvZQHhO^CY=>p7(oi-LGy{ z&z`kFKIo{qNtl-;5_ustWIO*TpVqm9F^d_LK`zB%VP>uuEV1!7k*J>ZT)- z?};tbY#267bFgbiE)8M8S^{CU@n#S`%|CvOPkwy{ED~Jbitdb9(#48lF{QF<&|Z0< z^paVpoURqgD;lUNFQ{&>kUb2P0*+&@^ zc4jM^=;2c=#8oFLo1{HVn6*rJ`7Z;ZX368ioLROFhKyLGT-7ZI6lj6LFqR~J9^^Q9 z7J||TL(2+zSUF#GQxz?|Qnb6*D!#A{3Em@$(KJaD;bX*@h#LTBYzGnScMAr8E`T5W z0^~neARLGQ*dsq~<0OReRytV&Zgqv3CcNlehixxay^Za=1y3^(3SKo;Qa!8HtZCJg zwlsvN-*fA4Ypq(P(WmMc(uuQrAk)@NWW`v5u4`rciP+v#Q%`YjT*lpBo27J&_O;}R z$K5XW`6zAXp+Ox7Pcn51fU?f0Ub4gk4ac@69OXtKq1hQ@p8&FG$C3zC*46^oWe)hu zz-ia+)XR|lf_g(~aT$&&RFW11I9jU4NNkZ_s5a>?a(Sa4QkS_z!Y8|PdR%)~hWX%)?6S0cRp+jS4P?%VuWo8gS^5XRIDlt9b z#D{U1mQ1N%Z)k@-$$_7iLMt3B@eORkyC?MNY^j4^wFhARqW72K1JGX#lIay?TYLJC zu?yJinu&XWnFqWKz&Qut4~xwH`#Vtk!-8WnJ=!N-%ojk zqVVCvZ6p+H=>$?}aN2{9u$LmSPN)ZZQC*s^wh0;+vH+cqJo<)zPeVDD-J+7S=64^J zocWF~lpbiv(XwylpXV602?4vA58TX6-N%g-xz!<@-H48!Bxit$DZt$F@#F_Rce3K$ ziNO!}=QMjU}P*1UZclnhUB-yb?+~b zvQU~YN7po?|NHPktL3g$bJ~5?X+pUKxnhMyumSz4B=okhOj}ve@v-2k%7$(ZSw*6T zsa%b0?-5o})^Iid#%zqoZYwM~ZPJGli<2T)$*+?M|5H7C#+l$M!OOGRA8A+WS3v}Z zX=Im)Zn1Y5|KW4Ga)BwF1sXBvt;Ud2^WymarG`Lm#I19NW=T~pYJw8e@<;_mV%a{? z;^?3M$&3)~5}f~RVUB#J`x+@CE4&!Ca*Giqyzx+I<2g$BS{#_dbuC1#SQP|QO&9+> zD6cbVI<14s@s{jlGiTB=jIid6+Pq~NkIr>=>5zRLseH-Xi7U%~8R)yaI8lfk;Yjs^ zv@rr6(z3gH-$VWcxZxxEQ9S30BWNU6gF>+AN#eXWP3UyL;1I>Ky|iUri(c(=)TXR` z+MDc{4>$J{EafKMF0UGVkHzM|ZYFo=)NhU=qKx01i^{&}Qsr_}XoY#Gsoe#|)VXQT zW;S5XJ7QS^#M4ePNfvhLO@V%x_;?TjNnv}Jrx~^=W0!N*%LB>xrbB6;W!ahR#`<>u z$-<>=6Js~d$UE^>^4zhQ_k1HSYSZ?|u~N78vq^I_>^@-4K6I$Hh`%XLT8*%v+@=ws z1*(OiN~3$C#e53Avs$wCS5RowB(9F@*>Gt9)$DQIf}750(J~+PQIx`AFp<>4W2eg& zbBz@Y+A+`R5|CqgG;2$^xq-=sSIJ!Hz+;$UTXJF@W^knu-(i)SDXt_%>CTyZAh=-G zEM^@2w6hSZG!Y4GQ6|Pro!2ZIu9eIwZQ3bMW;-Hd%lG=iG|C!Ta&qW&1i4f zRY2Ogv=66M!TBSv7iGz=eCeg>DRj0QcF%{kT@@)hypE!fk_x)brrmFew;|rFZWXim z*2iy_^3be=bAwzG6U7^ zB?8-52AgIKVH#sLl^aG3;Q7{?gVkGy8@je@+XgN79l3^Gz=v(D^=x%;o6~5}T9Ey-19fe_DHn;|oe>(mA(QcLOX4Rz&5+IVFcFnr`N_%Pk zH~6NL(Def!d&_oPBH&&}g4-vWTO0AQ^He?)m>?Cs6)ej>5d-MnL$YuxGolDisMORg zWyWr;kCLud>z{wC;&kjCxs)530G6+)n=03Or@|oUAOGwo~H1<2?MZhcNT-&m<~TKsdaU?S&Z7-C38Cab4`J@Z4Hnob;?R zJ2ml|U=C^o4H5S1?DhIyNCH0wwd9G+C({h<{3**&&!BElvOeyL1Hph4;o3~&b|f}K zQh1#K^Ke2EUGXHLAc)p$Z4d~blYk;|-TmImU?D{Iupw{+OtRw0@I+0r&iH3v1aiin zH>B*%-~zI{L0tlYIZ5S_uEt?uPC!OMThw+4U7x|r_mET5aeacsO9R2*4qg*tds3C3 z%9x#P_T3dkx6z>zg8ExSFdHvLFcaK%(v@z8w^E&W)8uZ3Hxli5)1+?L!aE(O2kpMy z?=OiTvv)rtCb4T92fE$el38%eAVk6!?Opb8H4f(4yj~nD&2rg1ak-$k+lGO)pAhWl zYc|<+vvgWryxAPO#MwNmb5lLzT^}|GbT5YA5bie4HrYShBamF6M@22V#D(nks*GI# zvJG18HrT8JTW<6n=S^MX&`$TNGe5V=0^$OIAu!wl1i^g$;{bzU@Qncof?)mQ`Z4*( z^#9-+-&I0l1n)kFcb(2N4i8L);;4z-DGQ%@RpM!`W* z1<1C?CF%_QtwI|8z4o=P0&?#2lVShf?I(v2#rW^E5-JAqA4h~5w$DR?bsh+iWe1-14$ijMbo24t+=uzoKtN8ku@9r~T#v5Dm8H7?Zg2PW z7I7BAjFnaXj=ggu%bjgNw9`WCmchO#|+Ez%Pe=7I6r8 z76A-;;|UH5dXv|OgWPrrN^Xw^Bfm!rlh>Dh>OgMa3Mjt^3$^tO{50gz_6H{~bFlF8 z1c1jO@AngaLxAu&_(uu;CloM`gX(|lV}Hj+aDMRr4vb$91~?do`>)WLKJb4>im!hT z`y5g!?42Z)t26ZGkob(ld|U!cB9nG4`0+{E$k|jV zS(&+^h^Rkuz#$jOYvG|h5?t94LPWH1fH zZ8TlBoPt23P~rVAHPt0^@b?w6;8 zliMx+1v?LqRyKN$adWa|uj~2LMKF&|oFbH!s?|>A!$K&8os?-iYLJ9LJDxXw--#21 z6JvEV*zfSvj(V%pMA{`RbEq_^bmIbLt7Awy_Xl0y1oWbGA5G|!m+h$UvE!lzD%|LDZmo6Vi{MZ za^i^Fi@7pL?dNB>a`KU_c1Jh2hxc4;5rWW_X|U*-IkUx?dRo{9|umNQMVS6S%%6%CnX$p24`AAv<~0U8@_yFnvu0%L%!1U zP+~Pi;7O`%2>VpnQtYyXs?F@D$<%hU=fR0Ws!=~6B128vQ+1<28^hXdnJd77Z;CQT^pwDZx4m9AT!ro8`8thrOIH0l7rnoD7W|g>&Ud z&cCY5Qps;_mOrWviAFmQsEYzM^bIP(4levDbOgUuVr}Ki*a}{m;I@!_VO3i^6KtHj z;L`g40~>(OM#)?>g#jcLl1UB|)X~C4olgo5_;}xu`m^rva=9V1e*ArK+vJhRq=|^X zB>_6UJbVf^nX*{=cNU17F3NA)DR{b91TO79TV^u~I;hHZ4GC`0;n?MDPJ9n_yDW;I zt0CX~vyDaCCk^b)JBizKodU`6z~GPZCx8Kj-|>GX@^y(#BTU;-Tqo(UG3vLJ z0NhU3mtLpo`4gS2ST*Z36jO-8AWr1-^Brt75zQZJ{5uNJ6o>n+%5-j;w!~}wK5~A4iXj#XWT53SujR6rl7_b6? z`hNxWr2ygk)msqmc}srw*WYo~JFFXtWI|_siS@&HO~r@nY|v}p9Ht8PnFdUM;BNs$qSM^1(Rf%7okl-k9jplYNOnwe zUnwpDgLRHCbGbMTw&2^>LBg}&0fE5&$9gqrC=z#BoPwTm5bw^nk*X81$zplgD!Dgx zIi7WY{P~>%5cma%EVLN^Bh+_6ay;bvCDFM6=*6mI#@8_#d#;RF>2{V|wJ{%szbC## zgTl1Gq@C@T^GFJZpFz@wp*L6ThXAs4@7}A2J^vD&?jN21t?j5KYoi{~@knzxvh^#!leGvWaY$u2T6mlKJx2|?@7w{CxEOc`x=*f= z4|7NE58RH`7XXv#+u58K=mu1_hmVCDY7Kybt!$6y4+U$Dx^@Nc4~5G=E~MOVK_IT^ z;OHVXdE5NTsYt+AD2R5=0MlHYX61`irZQT!Orv3&?7WaoW0<%n^W3;&Xe|@2T1HT9 zjE-LovLFcrupkN64}!cqqIdr}v-8d{FV)TcE6n%#;f-9qZuyYu6RJY?BLyWXCBeDM zY3`@d`FV9i1?22z7u$ucZ=>hEIo|a;*wnha>>GY~JPq?UVNO{DyPNCtfIdvWN8@Kf zd%6vU$Hf@I`r>v-Lj%ixwsF@xyn{QT4Z->D&q8DWCm<9!Xa@-EcS+^N{_p=B3E!UY zwXcQC{q#RR`NcyC)L6)>v*;(ch__pJA9difA4mrXsZ|4qq0Htjsvrp@F&akVEf9`Z zB;~1zP|6+?3@DYqrhE`i%lLHkI|-9W+6O2m7^fwUkk#1>$t|~*3p-i&a1a=aFV;*T zOC3ncWTyV6xw6E!JCxi6n2-I`o_S#ra0?V?A~MRjx{$4eaQDvtM^E;*bd-{`GkC|5Ys^H{bJ1rcc5xRegAX;8E2#x94!Snh1 zrp^_w$gGHb$f|&+Ztv&gyOxfjq!JiOQC=`|Qk*4VQrtopX@})5XQ{{kf~xcx`V9St zn*h|KO&NAUQS@L98N(bRV{U#~Rm(UnpI1fE;RpPzmw||G4r)GJu;)9JX ze#^u@gBmxQxar@joq3kB4O2*ZHzbv@)e@m)D^;bFdZxD56z#q~F1Z*zzCO!W$E>1u z=^NBS+fsl2rB`Puw|xG;P^ir*RO0xK!b;=t#4HIt-_VjV9o=0850Q8y+MfYo>+78r zY71h^^B5(&33kLH@h|R-qcl@4stWd3SR*6y=@goaXW#()3`fGDe5XIMf(xefoL*kV zA)N_`=qAdDu}sds`*-in;a^+tqT<)xpK?(H3u%dYh^)4#Pd9WG-hyrExR20TULOT{ zdjCkfFuP6>SW-SG*ZWHHy5XKC*J28_n3u+5H;XwNVZkVpGGkl0{LU#-!baR(XmD`gNbGSwsaTCbKKUEwB9q(mPl!=h*PsPB0XZ3Sn72a%bG4X&kbJ;a6_O` zRAMRDwW;zUYHB=XWUbwRGHRh23`T#bD~eF9dKy*Ta2Z$JK5Xb@X!%l)@r(uSz`&+W z=ptkb6DQAEj#zZoNrl;#b|4*#`G`+Ote3pVD1+@ZoGwK>onHBagR&UfasmSFm~*HB zo@hxokXwfzUO^`FlOL*)n80Wx#UgVc{|c^REXjRoSYY;x{H|8>EC?Cz4;b;V_x}%p z*n;jWdrFX^FX3*{Ie#b`zjP#M4{K`jW>xzI#Y4}OIe&qJ*m+s-)p|6~d>;a%ur)?j zoWu+|Lz7LeAIkZC`WHCw_lvrXo$Uo96W&Es#Lx7_I-r=YpmV|R=DN(oSo)91 zvszktmOmpF>(Z}SQ&m5r;y(hMb;swhyc?Bg(l@F~Y^Qmd>t%`37TV#GYr7rCbxR$@ z_FwHjd7C)%Xko;8hty6b#YnmXzS6w!;5$VJ>zYS^WGs>dlB2of$Cj(JqZ>#6Rk4-!~ z=hf~LZ}Z&082qJI3YG&1@#6|iJa1*9RI#kQ`z3FA@1m&_wlsY;>`Jw2*7D7_tGIt+ zd@N9Z(|dGSvz+PI+Rf-bT2$S#=WK)53)_NUCAMPC9keZ3EjpUmhd5Jf4)YE~XOhKH zh!)HHCGE6@KbBvUwh6JbU+QPsi??dF7oE`3Og;vkl!`_KF8bE<|9wWl{78KR7>$_P zPYcC{r6hVOQ-v?hM^>gK8(CFRrPEQ@LGxEkl=I`zi|u`8?BBE&%V@f7S-Y5{lKR@^ zs&bdS=KM9Ob{DTF=}6lWxsbf!sj>Nu>>AKw#<)&mE>R8|^Vr6Gmqnge4S^}w|Ix6D z$&EWpB1)hYH3s-iXlPI(^%tG9`Rp95zey1Lu_2wPRbP^2KW3s{fLL&IP{1)C{ma5A9#E>?we(vNc2nSD zQG*-W(uplgkJ_F)w|se~5MPd@RDrDI7)_YstbWM4!%!IDPRE*$Q_i|PFpI?;>_?tS ze2$5YqD?m#o()fo-cP#AWbkao&r;&)t}IrN$}>%;Qb|)$oJ)FgSO`we_}QWO()hzS z+B8x~BuU#+YiiWM{Jb8gA7(yYVXKs#v~YHq5hXi@O-EkIK5>sI!`|NtLFFvv6fmrj zW0^}isvs4b}I=Zx^Wz@NQ`T8)eVY6{bBlAp4o>qnO&G3QiR1p&P zEJvBR*I2rntNZ=*@-$s();$(etau=3p+IUR!8*>xF8^sGz(FMqHU6&rX*#|7A*9|$ zpBi&vB_1YoYoXXmTq2?&F}e4Yb0xx|5Xg{zXjn4hHA2a2APC+#94Yq`;~9aFK~hYE6Y2CV%L3eD&{@hORv=k^+&Yz!do zjUHN*E!6P0CN|Fl8o5Im0yqPmoxU=QW%gJ8dE-pW>>tnF)G9GQh zV63OxB3;`3Q?PFF&o*HQwzFNdFKxnbtS6<-L6U)X0lEtrT7=O*A7q97Bn52(bR&QN zO2ru>3=!yR%}O4YEjigSDq2pukIlH%wR3My^GA)S4Ig+-A&|8=*2+h^q4n1^-SRg{ zId!If2Luln5h2xQ3T)DTjMjV?kVS^iPv*|d>UMYY{+#?rVI=Pt(3aE zh9kp`=s!0SsB@DOCCgLka*Jo*aag(U=5XITXGe0l?+$HiwK2XI{m#EuY#gNj zFq;*2C27ISyG>LUW;jV%tshzqUBruLfzcQe#+4jDl#laK7ecBk(bO`q&v4y$@vD5+ zqzquN*SyE}deoBjQMKA#u0e%$PDpE6^(dfm0{UH6o&ac9Ag4*iM+YeEpRUxDVwROf2(aq6mu z+-8C~dP_gQiV$Cf;MRL@HO~dkA%-WEu(bp64Yo_}1!Gyq#E!hjYq~lZ3ylcrx`S9H z3Kvn#gy2(2CJyV6=N#%BIP`6qaAQd&u`5)iJYA>5+Qk`bOq;lz%f9CuSr^fbfvGFp zP^`h^lK#rKe)q)5;rEjr?us5CW3^h=NI};CSt&qU-n3_aWqzXV5FM-!;%|A^Pd488*7hn^q7T%yKg{_$#ox1%>j6qrbju1lz0>~jjc zeKAlWs>Yx_gDb_@s^9w9;HO0+B2*x9Ob??+4hv!+8`5zqN}?&(<=KQM-G9^W!Q(Gt zsUZnMON3JG+4LaIzvT&8IKBH-B@|0gPS!V_(%OIg;CcNFV2|2d#SH3!78l_rFu0Mt zYf~x|UKhvABuKqQr)7jwRp&lPK-%O;4fHhVA6<0T02dZW#tAHF%eOCAJ? zeAC;LTkHBStRH^f1S8JQ_!lc#v(@VeWHRUoT+W#7!qVJItd=8$Dui|5@yMARc(7T4 zKPMTGb%czZ8OWn705ehF%CkY5$>3r_dC0uk9NKW_Kw>lSO@0hB`fNjo$2HzZLVf@g zNpPbvMp#V7h7cUY%$XfJoqfTj{3hONf;^BRhU($|}Q9G*&a?#}Cy@YUIx`ji>P1P6kv? z)Z@_N?Vu4r{G4t#g-%SZ1p5jdeP%apn8%;SHJ`zi+7A+12-U{7;&5M;U&g?xc zR%dQNGz(|>A_Q3BsSAko7%!zaLwzdo{VEU@{9m>-lp0&)3sBQknOcfFae}rNP%m`W zw{OB*W|yBoS?@lKOnRzveyd*~e&O4b?FDe3W04XOKDvI!A*aY?rVBt$ZxZZq6pI(g zRE~t$Moo!y_dIB6*5h%J(Cm?{!bFo7BO%c3mn~qn`c(Cx@ipxsw}ZUtysV0rj-C%s zN7l1_i6#SN@^ly)iO_71NO^B1q0%8$>=Xnu9`s92y%@1eO#{>ZF-9C@l8D>#c%o2# zA#oQaA|vS{Ur$Gi;~WMZ>_l#gS>|ao9Pj>+Gyg_g8b|_PIkIAvnMlyq((A};)^o#qrVyu0#iuIdRzx*RuhnA`?U z*&ulvaT<+m)mm*1`$9zPTypPbtuw*8?bc!bEh+Ze6KZZ%b9myl)xj7i$5t;Pf?!f6 zVAkIApF$mb>XvyNNs#!KU^MoFRC?9AMo8*1*S_-A;&%^2?#e^?0jHACk}@#Bv2RCK zuE%H-L`2;DK2wH2Hed}*X>g_x!AAp#C9n&NKmV6>1o|0F!#mA?b?=#ZOH6E0s`)ej zFGucyTDH9rpdt};Y%9#j5^^0bi4ZXz341mx63&2*DV31!^a~_!#W`)oIBtdiO6Rcg zglo>Vurp#RvH9g+rd@my(?fe@V5G{o3(^W~KFlc+1yVrf|<*lxu zz4A;2FS-`!-3j&0T&wSZmBEXiT0C&_t~N|oyxT{-s;L&9JKrf|ae1 zInBWK2Pzn5vqyY&q_ug!Z?6~ANh96&&gs}2c`RA9=VoeK(h5$Uhh>iUy9X$Wv}E_s z8zJx@*%<_7y0?+_u%4C`mcodgI?d0$VNe>|8*`<=#WSEB44uk$Jmh1N;J zf{u=-RRT;m5F~M?MFq{u%nezZqFn#%+)7z3FrO8ctF7n^D}BWycT+nCBGMbsA}FRE zFfDn}i&DekZ-T(4Op|HsYJk_VI_@r+T28j(oB`Wsc`h(MgGGD94hKS%`ZyaOy5&Hy z-Gsr%@e&*svBn#!=#9XAR|;R>hzn{x+no#U6lvtoDs&au%IQ5k3<;7;w zvCA-7N)lq-=%b$?r#Q{9{yKesjvEzoNE_4+k`6H_WUq~u3^32F~(Pg zRkG3$nNz>>?l}m$+l1AqlguIcZ{>3{Ih7y)W0=IC&IpsR%Q&tl{vk4%wZTEgE+0`- z3prZAho;o8B>a1G>vitb)el#zoPYMMbvCgv1JO)XCVN(>*VFHraW7w)9d%;3x=J66 zoY$E4P>6mBo1xZjl@UmJSFkZq*GSfw44Z`QJ?;(}N)!^PJxZ9ABG45+k@P5{TDMor z2qp1wUk;HB|@R<2s5Y14>$ zNljL%!P55=y|Mj_Z!~7+rN}<<8;{bwe~ivvClT5mz5lMi9)si_I{Oh-xoM`f<<>GN zIl9B(9e#a9EqX`2PEC8@=N=fnA;H~9WtqdK?+I}y{oKtn0-L?VIDLg6d55(74q)T$ z8+-+=D`gWMm}uOWXxx!%+>(9*N&q2Bx<=)RRJ$U9$UV?UZ3k7kc=+~~^YT5Nl7#Ur z@4i+h?zZh^EMA~gp1p8hKW#{mw4Uw=b~pWn)m?n6KD!E8sXmR4eGvpzgM49ZZ6F(; zqV$1JB;aUsjE*}kkGgN}B62KG$1l}xduE3YKA$;&-z{>L%4Eysa|-s=Z~{bUPvVS5 z8A0?)!x%x-Xi?ck4mwHq=9+q>H92-V`@8$4oHP__btUbabTZpXuMxY^gMN(DND(y? zmg+FPFF%qq4rLg+xNsSHe(9bt_>imKHEu8c4IRnGmx~V{yyN}*uhrSUM0FG^1pahR zjo&>BtOrpY%7VfJ0uA7wY^i7|^Lq(gr6&c(jlhGz6I%)*1ns^QPVIAGEWHv1G$p!%kk_3k*^9E2T4{RlIHYKoIDdS5=V&*@oKAHQ zQB$na??dJr;a#bscaDlwm|r>f&G@vW+8lXyy7^P2)2*(XZI`zy zJt!2|)0R|8Io7Yo_K=cl?&Bf_Q4CWkOt9K>3Mv?xu1^J35Mq+hMs|;N28%>uyel`} z-*6)mx8qnvZmzm!Q=mi;0Ya`s)wWX-eP+3BsW`#qU|P+mjUb6Cy$E1#oE}XmJtvZE zxaCjtM&G7xJYHstC9102IIajis^90yL9ebKQ_r#Pq9M*n z7)mgZ9H4G}No%zH0@U~G%EzEA&vf`F1=pyp;zJ-|QzBGv9~P<=K1$Hkvln6tb}EHE zR4e&x5HME&d4x1LPf-H=uuap|A}{7e_n1mN(>IM75#=Nv5`>>f^cg zqa5zc!JnAURdWqi(V-`V^li?LO7IZ*p0dCQcxhloqof-wjSUR{NeJr_cPFZC`DxWI ze5{o!^zw4r6=R{+qBxnw6$FLp=x9q$GQEjV>n#p8vB_|()xn#&8~?DM@np+%&yRoW zczi#9!?FL7i|BKg!EBuZYG!qK3_r7gQw&feqg7IFB_5_EZwb=tw2~rwow0W-X@raOkMXPe(z)jF+d+Pj1?vy;t^<&KLzn8_P{Gto952?0Ak^H5)K8~A=(%C>t% zY3l}vgcR~iuZc}#5^b${aEZ6WKIQ|{;fK0w(;`8y1%VpV8Uzy4)*;mPrbzZShZW|2 z_I$34*e8Gt@2oQo1TJiks1+so+w)hgXOL;vhwW-x^qScn)21Fa_7ql;#Sj*E76vV0 zHB+CucW1^X<7hGV5|hVneWM;-*~HZyPC-&6s63#y*h>ZR_YJQISRzg)c{pB(T5605 zmm_&K_lR$tf57reVU6eVcBktdUBMZdu{p2N&e3Z=029vQh)ylWgp+y&Y7VkK zq$HCUb4W@jH6Mv~KscNiDj&K(6VZN!OYt2n{B3*szs=Q9E6#(3Q7`wQVmya)uj~J| zGWwr48?OI$xHpS!dqSVzrQSLSm)f-x)BZq6?OFY;6aT!9VEc9D|EJym71(DleXM$g ze)y;VKg%8)RK=5Sn(^YR%A!!W?wmN`o*n3)oCuFuQSBy~EmB>ZylbrzYV%?~U)E|sKJWPX({Fz%qc0mP;ICzT^$ta`jr?O92wGF3 z*(lnD|Ks#AXqIbgrkC}b->`lu#pC1pGJyTL4{M~G;cfTxc6D|2{P@z_@qXUFc>eqE zYVoyTduy+kH`{ks_a{x_=;|tJSFH0Hz)It%RPdUe%q;p!v;Rx?-xWx$fk|8JsytOFM`Nc ze0_%bVl5C3zO25qu10gMoYclMh;eTXYTHvzMIH%bW_*$tpLa0*1yMi@ja<1t8d$&J z`mpMy>_m-jLE`Mj1X`;up~WGC&8!42N8-rcmk^b(`_NFVf?jn7bbpf1+?lThuUqZU zDYDO}zqGBEJBl)&KYn54 z62!PKBe#!7av{3WPdIfEq@c^CQrPR-l<9v~l)?6SC}zYFESQ0px*%Qk4xvS>k*_y2 zyh@;X_HRX7fBmWEB^+6vpI5;(OBKapt)kkztZNO|)n#~$%V21~KTOO57~xl%!X38i z)(Y${=@~}B(`>u76KLz!&E}nuK$jZI8QcrGZcXm%p`Cu?XWE+Wo75*OL*F6>j04`0;Ft>LkygSZ!DwY z4^rN;&5O+}7SditKE{ogS8}s`Z*KGLHDz}p5)wzg14v9x{3_ohidltiJ7&hTnYt7DUwJ0x z8`5VPo-t+6ZwYnLI_izZ4F5G3KQiZg>CplWe^GGl56LdHR0htp`Fj)h2j-x5hP zQPte3)lq8(t0`#K533|v+pgfmM5yzCYwK4-evja{OkHPhIvqtr)fwd<)>hN%ZAP~hp%TN*RY-W-O! zZQ3jyS*WEx3Mo}r({d9g&VTdLQe(aM{E5|WYv_J%7e4(N*apW>eg&?PO1 z^PsUP7HyM`H5t%#dp&0K;NIQJwlk)BP-y@DqsU0moQyraiZ7)^1RbbaHni1lABk(H z8Xo3IolC`*uXa-z0iA$mC8Y7E@!t%sNzWF&CaI4F%d*u{KdbU~9paBZ3imA-2+6L? zz0k!Q$6vf~o^Uj9zJhtH-}FqZ{Zrj`!qem8Lv~rqZSj)chd1Ur*J&+z{7svIfOG-B z7g!69Q`y1dYVpGz0)%PB$luLb)`0TEm8JthpXD)DPvC1vbv>OP#?K+~ygYcwZEFIj z8QsXKBs&z)W4md{bq*vP+1RdpEkP<0Cu7CSlWmpZs&6x2UGeT2)qmm5e%FiKaC2!m z?4eN-MJ#X1m!T+C#s%UN{=QN}A^qo@$94WfYpQm|^x#ZhMmn5D<#gY8*pl`MS}|lP zXYu|rC^!A80W_wN7sxd$dK;jXQ!oz|jv>n$h-*`9&chJPzG0RUY4SEkXp~J1>$JXJ z@05}c7#u7<+(F%-&uh)XpHnq8t5 zCOOsQ<2qKe!HvnBFP%lNGb;vd$E}epiD}2J;HD}0;$|=Ixu1Kc+w?M06^CS?7*%vn zd z;vq-cudb6H5gbt8pheO!r0&i`BNc65I4wX7R-_VmZW#))s@;$QaAzTu>$OtEKie}Y zwz>>F_fs&%<05I*O&H=KVr4TSNNv_B&KNXgq6Ylvs6y|_ZRJCUYAvPj_L3MdSnBym zaUl|?r&fd3RR6d2eD$}o=Qbk6l*};u-8}d+&aOCJn*LCK$ zQ#<##pR)RQ9pNuPNKBzhpv~x!8;k=kksW&R=*K3xQ;W_Hpr&n3x>8L3cBHU!l*wSn z5la{Vr2(XR$vl5#F;@+Qi+B;*$(M2kOk4A16kWPDW-^0E-zsAISJerlj-hxw#AJR7 zLs=~Pk8HZ(3GQ*BXbHy+JcLFqjJ$_DCSaG>dIbm`XL*C4Ujmcf z2mZqk&*f&ERDa0M&e|xKh-^#TEsVZ^ac*oAVU<{NaeVFbrqajC@STl@+VP{6U`?CM zWyv)h5J_~A(O~t!Q#r1Et4J(_cCuzhqe7{mV&4A?SN$UN5xJS%!#LR^o!bUBz<$%l z%TU5Zygl$b@9g&iUC&ssL{KxhEIOYWgAsbt)KPPIK0dDNxy`nU_O&XoS?@p#<4e}G zWnZjV2fcW+W=M1Vha)CsB%Pv1QSRC_Hyxn}$;e5;LmtVzL-2DB`^

    3=leig?M4h z@RH)G6!20st(bRBrmqZi}!C(nd6^u+`fNPm6CO~>lY92#mb;3o}@pXam#1@3R zoYx8ST;feBH4WpTe$3rjf-c~$ltpPFi<@8GtYoPJVWC6o)B>Ak;$s0dP5Z$HU)X7Q zn&r+tW|LAe${S7To%AO5r1|q64-UYGjXv{RL^laI%%WtYt!n>l=kRTC&#{V~kmGyi zIZe`L^4X1ny!}DrxNA`2U^IaISb>Ee;^En|ATR#pk9hCayOB)tfIOVxTjz-7O5(#` z1~*Bn#M#fU{E#E{`C9$(&pGvy=`iNe>Lqfz;rt!zt!arK)%&5VgM(vq!FZ_Q)-WYf z6pcVtesiAIQUbl3<$rp|g4%ZlZ6IB)o8M~zZ^omKu(fJxKVOj5}v*fDfo^6(QUk7l@j zV&=*6D;xB%Bg#x~-#M+sJG zfkT*qUB0iyyA+2Ry^{ur*}LeW2p>@0akjj?A!H7aBvqnIIla=*Yk zo%g@`ycijXRb9o2Kqk0XqePV8opJi-l9H&Yaw(n$TaNzI-{0;UT=;t^dsi^9vsbup z7I+nb!#|7Ha$l%2pUekw$^+s?=Cz)I-lW{GfK394B~qu=*s9SVtEJJYisBY{bK%z@y<7^mYHRp)=TQTGtm>O4Tr+eI>e| z^TeQg6hpLwz~UB6PNY2V+-T0Ol|TaSjtY=7!{7rY@))F874WUqJP<=Pz?=bHhsbT|mdlO|yQzsK0HRtUV2G$+yj=ao2}BJ&SVd2pZqqHRx>z%tehupg!4AJ!ko^ zczvDTsM_$;fiviog0(RA3HG&KopV*hqW)|})7*WHoyzg!Dz&KswW;kLzc-%hS+>g- zbN%U4S7C5@wYn{$c;dE0Wjy(LgR|^mF4%CYzFWA!yN%EMCEo3#41ekgO(pNH^H6%t zR-dUlm+$kH?|^}zfxyoj%w5?f76#J2qMe?tsUkWhnYqCd5}L26{LgOF-poA;0a;s~-ACeV*p3xkbVXL$`TFHZRp~mYTd$R)!smWvko^jA_~{boJ0*p-H&Erou~K zb3?**&GL^8sFCbAk5C6B43B7s!8CR+7&p(`2)3-4O%Q>iP*h?Xb2!=Hc()%)$e?M6 zo-1*CFf36ira`N4Oh|!#6i`zB0MI`v-^1)fGUbJYy(!Jqg#e2j)fOR<6;oc^RYMs2(Uh|6flhXtmbWdQFV?8)trzm-%_vJMb{xI{oK2U~nig z!pywj-ZYDyE9aLg)QJ@ZY1vDqO&~!E^n>^JzjeBP&3rbls^@C6YIj@M1-jHpH^JSV zy^%kP&yJ~ANZ-au{42>9l2e});liR7Fotrop^U#I5H_5(%8g-QIX+yct*PyOxHenE zzLzgRQFz$*aU{n`xg7mE6(ES*_AS@9TE3NtU|p_$M;4Y2Zb|z3cJWAtuTKka+V809 zbDAJ~dfjuK%hX zLs|zEK=39((&FMtT#e{2fw})5_UJHDWf_KJ1l#->9M!t_*lCOOcwO32u?7g-sq zwE@&(xKvF&Mgi>NdyFQ=9Dc9aIBqU}boy6L<6P0{+LRXthER^CJszG>8sBl)xH|bo z0GKBnDUi-ets>2P@nldVQM;#ZJg+gcX|+#=j*m6a0$OZ(ww}jgHZt%ej<-iIy7j(x zCmc2P&RmiATkRnt24ES@`=!j%bP{Nqw=D<4hNq<-H^dVl)%QxAt57Z>!0ZSbDpsVI zhvJ{|5T`I*Rhg`ynFKxR*{&1`=h7}o%3O)%dKz7Hqs9=r)NW&$*a_NJqRA3s2jf+$ z=3;!*{e53#iGX5f{p_s*K|d^-D3S9NrCKlbdbgXEE6bRtLRj|+10B;fQL6~@huO#o zdgrX0xq3Kn>(=XX)?+^-&?BwXEs4GHI`kZ5elWy#%8_g}N$;DDSMr5DGwIA`=~4B8 zKozoGy{q^Zp!C0VrA9X=Tystq0l@to#f7mM!R*ZyTx}5f+LU`+_2R%NvTO0Fbsa=> zvOwep95OYU6{9?(4aB$(qz73fu1A904HApXC*tSYP~M_8G< zoV8V(*{TH6DcMer$GEum0?x5rtd?t~=HuB#7^qnnC_82zSs?Tvnft>~CaObNBBl5~ z@jtF7A6z-Kq*fw6{Xbhd} z(Yj8BhrnY)ytD5F$u>yR+HD~&;{OE(=Z@a1FkC~xl9r9ogNsfxlmy$bu@=O82qM%- z$O8xB`dihCU95O=;ep8LTA{(J-PCE~lH^B0vOfEbe$6RSaj`0 z-|1*`@LaHPRv<}%9TYdTN)-yxV}^$otUQMr0MK{M*KaWq!-^`srFGA!;xGW0CnTrnZ!U~wDNuR3#<$H}z?U9|V4_w_41jh~q;cm`rgW9)ny}ML2hlwH zTo@rjwA`TNSCb{CBonDTJE-~tx=e6-W!+Bd;@i_=_PjE;-DfceL=-U)>(wU!mhb<& zg8o2x>+XvKB)lw*0sRY7!R0r2mJN@XXQG|+F2a0 zEYqa0QCu(2@#3b!w!};+M(iRJVgx-kxyeGDNC?(|8}d7b$wpHe|9S}EB$*CJ+zx6f z7H$(mX3lyoU?CV203?&(>aXKQWDLJvT+<1l&CK{p{c77pk>vrTKiO7mA!s=$tpMAo zxl65#i8WYp`c>CKc29^}K1=N^I-xL|_jujY1bK(xzxQIlR`==f1w4nF2zRYz!MLI=b#W z&7lTq4H6kt4iv$wwpIfzNv&wvK3@WExH1wbdk}p*P_AvZBp1jd3WDcUbs9p{4t`6T z%IRg;1!j!UZ&J+xX|OZltEIAld={_3uEqE=I=^2p7666fQe$ldz)g-xeor zS#iT|CI1ke0BeSit@!0B)LGZ}bb6cFqQFqpB6QgL@Ik^zivk%7>=WYWdG!OfLXeS! zZnxZwH`D88D-`D_!RUDl;r93QQpZFKVf>~+qEmqMfqmd@7TY-a{LkYFR?6re#J_i= zOlgxpfw$Yy_F-POOi~^>(lY-b6j#FaOR0ap+V3GqG1%vb5b-jmhY-c~$s!tawnkyR%EF(d+rNe?2@r ze})u-G$}MV+o4l_aWVRZuU>;|X5y-9v`ZGJ_cK|@@2Cr2K1+8IdX4mLAKqAtI8q8th9U}E62DQSV& zqN^jhKlpIQNvO+x7<|2)&6%H}_H9)JDrfwD@p2fT?{(~cw)7@s+1GI6nJ(?f$aS{T zD0uJlAjD>_;cFgQhB6{GdH*_uT;We3ADdGBDeh)^aYIZz-F5aUWWv!dUA%kU7$g(U z^M(ql1I1cuUfT{ub)4hqrg2s!ZT(2q2R|h?HlXfLl>uXWCR7_c%vhlmwTY3#Aa1f; zBtP--o=TukPv2GAbf_G+cu=Zg4rYnhUS$f``}&fe|DFEvr*DAkN#@7FH`Jl>(laHc z@7Kin_g4=krNCEW!$`-o|3m>_n*`WB50oa$0D*%=Kdf4GI~B|t41CS{uZ~wO8gKW* zZ_SCjJDLdFn@%YT-SqKU9)v$Jq_4WijS{3!={8KTqbX*fqQJ4A!>X8+v_!izk`$k8 z!}7K{=4d~HG;My~`H4UoNK%EzXY=tZej>q9_k0%@Q#}O%;TCEjt3N4ut>UHOmZCrz z3;TKk@}5e$(Z)0R4U(=TwvM{;)tUFpFsTX}+}l=SlLzd8PQ4qKAa*aJcpn^WAi&=DGjc zTLB5ss8KC>kVlqF6|P;NAi6N(WJAJ3^GdJIVvTEdG&I0L&!hL9buomL}ncQ&iuMrD_L8^EGgL$$oM=RnfZSulKan^ zTb>HGh5*}uD@VQUj*dLz=r0b>pep`D8VCHh$4$|{q+E%^(Tg6l|74OdMfdfS(yygU zdSkiWE!IY(;4de6A7LSqhr&~$*Wsm?E&6}FB-ut!>n)ICp(sa?`Dqat;_#fqP43J= z&DKQl^raAukDFk zvv-A=$evj(8dWdoCDG7~TK_rE&mG6uTA}kW`{49!B%JWJZq9So0@KrJ$GP7mYVARy7GKng4e!-TK+pItn4-x#KG+lD3+u-1!lBt$>Qo~*!QG$s^ilwIY63A ztN@cUjaMD&xf*6MO7a(axXkE!u7+IERcTs-?3gm`4l5vp1U zB#3bgw!oiCmW=`&VZ39Ug23=JeMbBNi0bv_=nY@tjySxa`0fDZ<=|2?!Pn3_^_(Gk z_)^XDM(qiP z)nNHgdYanYY;XP<2QfRTHO+W9d+ftfxgFooET%wMgWUs9+nZ%0m5K5?ii^&>=;~IO ze+5ZkMJxkI^*Ki2kc!a!Wm6*O&jztQ;1hO(hadTQ?M>GmxJTaXXyVLL!5!!esUyWv zjPNN=3^pO*GgGy4mD&tot!W8;Ol+!^Wh5pSWE|;oN8Wvm5~RfvtAkl+1UENY-wu8| z)WpD;C?g)sfy|ak4t)0^gtqFpamPw}OqP#qRRwin31git(P*})mW&$D>!L>kh z5`Cilc!qjIqw^5#&%NpmGl@1Ghk*vy|IHo^v%u(~pK6i?t{=5IFN`hP zu;%%2x2eNFR>lcO9e6G)QZ67Y880eRB9@oQQFt1Uk$sc77rvbx#937)jFdLW72hkf zRBdx}8B9HTK632*mb^Q!Nq}Zw9aEXvPvoJkA>iS1lXYQnrgjKViKiBbP0qFdapQ2swYkf%v5m$JH*r#S6=5;rtaPqm%x-%W4Fdc1;`QH_Cd z@dpg()EaMFFtu(*f9IWR2L4tahmIPxeLc@}&7=$MEXQ&Z#Bq41_j6vL5asb-oV0sf zbHcZEt(8a$j{&R8KupZW*0us!Xf@jM*8p{1bO%W$ax(z%y(sg;519m#a(8-K={BY_n?)#hP z_G`Rrf@`V%BJ#uEnXn4TIFdMYuW0ful=bwuen&?6%Q*FxGW~|Ns9WaV+ai6Y%~iSh z9lya#OKFs21CrvEYzZ&Hdj~Hkd#XlwwHc@zv}_#(FPEn{g&GZYa4b;$xuQ z(2Bm|1p7RzhEwN8wu5;?Rj8;#``YeW3J;VuT)KZg*^d(!Wu8+>uWnwPqxzIyMc6tb zfVs5h@L>jR&Nt(o;BE72#+VlU1pE8R-9b%xf3m#n-u*=Vx<(IzHxXt;xUDq_eG+IW zbkfudDkPos=yLJX_8~gh47_VNb5!n^lHu}W zf8!~Ptx0LWvsecXfT=(qRN1}FY-q~!yoXy=XlfUI<=lmt@!+Uz&E(}9BT0vMOQA^9 zxDaBZ!7Mnhf5@tvdS)JT6cMB#phH+qXqRi16X4fV#gVFKvpunD62d88bT@erz;$P> zn+s>N+)GZXTTnm?jtRe^BHVk&vF%K6!>|MgLcydg?jl6o~@pH!6y@;=#!>aP9 zcMta~yw%8m^zT3V_aFWHkN*8f|Nf(Y|Ixqy=-+?z??3wYAN~7}{{2V){-b~Y(ZB!b z-+%P)Kl=9{{rivp{YU@)qksRe^sgMm6i7+w(ziONt=@B9UBK)h-zaK$K0FE>UprkC zM(MXEqs1TAMm4D`%1AwXMU?h@ol*w2ueNrD-I1k z7p!+ZRzP5AYd%Rj-mIAmPIg|Ytq|cdYg6W5Yp9Vgs@LN|3PL1Q&T@@1UU?^T0qkZ|+QMBm(a&h`s@A+N zIlf=lE=M8OZ~E5dYxPytX+8`3mB8tIjYHelfAl>k2N+Dosx`r%7rzL`=x6b|P(5%j z(ZPh)-Q8*NC6A?&p}dra(vxh!ZD*LDINanH7dQWg+1TC>Yhcp3X_iK(NIt~Pkf&tsq*UK3z{gZh zCBrlvwx3YOyGwPmLG93mJOa1DYuQr&$&Rb<_HAD_sJOrvUsEQB6 zJU~1MrDS`VM3k>R59F-X9m(?hr^h%YD?^OVsZf3HPWZa-^KD3lhS}p4d%c>)rVU{2 zei%q-kHalqsZ?d1EF;Mv%#NJn$QsXI&sk-5XsJ}x37^D1oeJ1iVrFjuZJ1pi4Ds|H+|% zk~O{dxrySetgf`$D=$}LFwB`c$Ytpb(X(%X;zJK-Ngx=wNhcP!k=>;%h-tdcSlgnrg*E?fYe5lL9fQa8>jci~q7z?Jg03 zHTNiXM;Y0Lun+H80 za13q>UTHP6hC!k{?QMIZ;AQ}?#bvzhXTXO6Skv6OIsL?IFF}|G`o(Q`e33jPL3Nz$ zUUioyCQzp&pVZ1mFrE(uEK2O*q2%kY{OyRsYm&pZlVGBBP`-^>53|APy)RLZ26-5P z?mQ7<*V8sv8(TpRpiSJ;LtPKYB#wjqihS9m0^?Cc4it&*P4J(@Sqckre zMCwQ7pE?1Hl#iD8mflYbKuZhoH8fXv23NkEI*mBFRyG9?xvYKAzE5NLlMT+Uk~18A zz7;puXFrqkJ{@ruMocQ!R0onjjD;td($RQjt=u{V_TaVufR_yuUoWSJ*W*k}w~A&8 zxEi)iG;@cJ>iE@kHj9t^==VT92vTF8&3uW=>OGHC=6;Dh*Ky(c+lQB2HeuL;yT=np zu|y6S;@yFekKn4OHi;~Kf!XTb0TGK>SH8>~u2+Xy&KZ6s$18u(CYVdf14`?|;W@d5 z6LeY}4yV9Jb}KI!kv9C{?QcaV&4*5|2skr?b=Kv7z^*_jyl(g!<$+%zjPSGIMTwoo zL+ej49$Lv+WyCiQ5XGc)fh;g~!RcHFl$ixLY`FMC{;oIQ^z0b9W2||0T`N4Tsr9rSo#w;>knQ1@km$8 zSo`bq&B7{*MjPIaJ;uNbM1@Y2dL0h-Ld&?M|L1zHu5H@ZxC*FqPVSvoP@7yO;Tkz* zIMzY~t%Oz$+L9&d5#NO5ur;MX=1SkP8|@yP2_SmnQ%Pk@4M=QOeFIf#5{IjOO?wDO zh7cH-(i3yGimR023->%j@zQ0!wagECi331L0qA(VJfyvg%mk4nY##lY8@7kz#5S;j z;NW=*HS^gdW%7?j2P%wr{K3$0%ez}<`{Aw-dGgu!hR67`V)89V0PJ`>yIKO% zz>z>YuSFsiKtMGvkB6x}tdu+=+1e^<$Tm-Ng6(2rT1jNbv?G3v+HImT2IgirV| zk3Sg^b`mv>UELWQH9M}Y(LQC}l$(Cx=)s$d>Kds62}A2CUhwYP*an=4VeB>0>B_wT zr`7Na#6$S{+$4HuK7K+^$NG0Ig#+}-mmALW!*DvT)%jgb+~2@$b?p{JDq;c#2r_Qd zB&?)t@a9)Wb;9jh>*Zk#1>C_x%!55BjvqR z`N5mMakIXy@`H1-{u|njr^S`PCFb7K+hnh;4;-VA_}g`(M^igdq2O{^ucGBVI(gF? zzmmc*W7tGr9|%WQEL3@)?B)DQW2Wum(yvUAn8Pm;L)wnQ0ca5XmnTI^0BKalC)j#u zTDbgHinW=r84R~e_Y-rl%(Vs=no3kXJp79zeRS2>c+ijkW+|UNX{X+?_ZEep!wr8p z!(LIv=CpknIs2`l++Q)*Om6;^&5sJ3q%Jk7n-eJqjL~41?kRe2e(sLyX|z7T z2~;R;x6d+|y#Z21&Xt^EgmJ`6Oe1(Rt&L_q+$f7)SZuLlfbOVQ$0G-toU;E0`aC9bY#LCPd>4}W6K#)>n-N~KsN5l8 zUc6FwA)+G~zw}wyT5TMB%_UO;zE79mv~=0=DFoUPP~;IiUEc#Y&?jF_p4>k)bSdtb zK}jmInT5`V&1+$xw#Q+lbY#(ewGG93d~3>>rIM`o~zuf)xL2bMlu$vB`iydUo&wNJf;Qig38* zjUzz!Gs4`_VXX|)4*=JLln8@#GoW33ZGt+12it(W-fuG>wpbl@oN>=k4m|>-qxytC z5W2O+cG4amX`z$k-UNzx-ch?RUeM}jJz5Y8%6-26Y1?ce?B!6iaT!;c8%Jovb|$7c zsun5U_XHH&!3f7Z@udvP*Nm}b;H&!@i9u=@$((z^o%D2Y@OBQn&_>YftR5xi-(})V z-ZCJbv8C&EF1nQiFpJ1JI>2#+40sS{)6Md1ynS2|;gm$O@XJxd5;ph;^Sn^&9hwv1 z{pyc!30WG~j#A$@FKKq6p*ff-(y&u%b_lECpMaX{;jj-ohHJ(oh+0y5KwIp^@G~(* z*%XcNdTBVc$NPH<2#l&ZT8qOeX*1v@k#T#OIOGteeo%{r@738rRE-N?W`Tt(viV@V z62pYLE86t^!fwKz3y>@`|3qWP1|YTTB5qi^w7A*`QH`X*5Yvs)$M@sLl{HsU#L~_uG=Mz4UFrjQ8|4$Y{?`hc`H{lZDHw z>o_i(a{hd9MTp_#t+C`Y9FXEqgtK+Cy|tWY8jUgvy5^a+Ze|~VBW`WEtxI+7U5~N+ z6DXw^Yjr2$H**xu>%Jc^%0gq*ANy~8InoE^Aan2wY#PI`0+BpnAX^K?T$JMgenSwE zu-Pn(oYeiUW&hg%DUrR~DJ;FLZAppsyzq%L{5sn>NQ15zwhXi;YlA5c+};Jl_ z_|{|D;$mtrJX5|-;S5P}J5`su+nwH|U1vxmW`c7BmKb_oiy3uo8-XL%;Ne%0isjZu> zyexEc2tb3u?h&krFHvKk`OwSNkHB{1aQzOD_;*A&DP}3&)!!~w8`il3*-m)o>8|4Y z#+v9^jQiwk?%V8-f@NVIl_PNZfzH?rrFas^cyp65@h4!R#78)gFP-5?qn8ZaiDLR{ ztM)ti?9w)`({w=c9ArN~6J1j>Du|SgnQm)xfx4H3S?wXyf;#Wv4*MXv4JpX`0A`wg z+LElVt;x-}%N@7Lw!I!#DCFO3VC)+rvJKTmspnj2ZA*Ly+(T>bGK^LEIV)>hxnh=wpw;pf1N(?3eYFKjRK$P z>Ca|8n1dtx|Z|8(!fm1{`!izYY1vTAAd=^5ZyVUM98Qv18gn@GO2e)kQHgmsa4a_D8=; zrx320;=W%U`rIzJw&6RDeG(Dn1jOmj%;j2n^9QIS56YOWWzS`{8=jkw6%$x7Co5x4 zTLoyNq%5`{8J4o&1nK1qAs9eLdgE%^bN{B26hsCEts`5pq8o5KF4;!#Ueo+s9l z^w&W4RHDYQ)2m;jz&CEkDZfdoZUDsqY`c|UET#^f{uGFQj*FC3BuXKt3^L2xA|Xwh zzXxdKwr7%baB%H-e~EV3OYM*Q6{8TkY*U0HiU^H!*2@UA)51g6v`1ZEIKVVYc@h0i zij9(g++v)qFK&5GVeas-my7?vcd>xT$>57wq8v#s6Q9Y`4|yoOEMVEn1`Kf*_&{Y%VDge=((Vh2cF~tt6=bAQ z;NrJvNuc>ZIBA}ea{f340!|bMP#2mM1TINRON#ORGzSbH{Pq66-@$1vf?FUE=Sh%# zS1eTAr!b^RL%E#Y-$QWutCZT^a56ha5eY9et5p7GDMAU_^z%ABeJ=Ojm*w<}yveP< z0QgzYfUw$$?VP7l^Vx?TL{DgS%b15v?^lO_HV0TE50! zVO_cMFe*h_I3~RyJ#*sebl&S@2KDcYGnde6?KaZO?rV?J$h@l?Dvv3*_`zb|o;3_p znau5*`Jr>H3DoF3?QL}ljh#;WL#r=Isa!wSdfMlTCJNd(0#&Eo=(MP-cE$>GW2(%H zwr1DOz>9t+#>LU!bebbyhx>Yo9x+cYWI5?5(dJ?a@fmAY8rqh$cgzG7tjx|i2B||+ zo83yC>Nb!OS;qI9glM>@i@LCPu`B8nQcL~_^3LY(RLJ~nDnA33(KPt zJU1+`RaXD3X*z9113`$HpPrOVPY|9}1ON$kx2Ts=GR^PG;%m-fu3=d?q8otL_+Ft) zhmBA~2G4pT42z4>6y_$}EuLwa!VNMdB{%HeAr59oygrkT1J2}BwrFaB^snD+rLqg~ zJ^HMDGEHb)4lval5cIc4FOO0%1+hk$K=N`&TTP672r}}|GjxXNm?mxy{%Ap2Q@Jm| z-bFxKwpgTI#*5@MjWZb4;IddAcx663elnT=^^~**~MPtgXX$ zGB2Yy&=33%T#~J#P2ltXXrzL%Kp8O$cjCBqq)`GvYN>jiA|J;J6b(pU!Z!c$w?0GI zvzQgLQ~Qpv^bqfE3ikNy+U2VvU3Y;vRUmB9gR8Kg89NnItqHt_NH<3?J;~R(nv|%S zVZ;p^67@SV?C@RGj`28Gw7YEaHS1!J1zd57GNM050s0KC-9-bDlw*xg$;72*F|aIz z&|bFix2D0x{q5@<)NY=yD)fJbtji#CR3P3l0C)2|x*&Nt#4xubq5K>*E>TUrH;$e` z$`g*Tmw&fpR-m<5Ur2NlzK)q$~1xr<9q{qLlIN4j%_v+oJcocv}kI z*ZpB^IB(Ol+y)<2yAqqkkto+)iLl+rtuaeqxO%jRsJ`x=Ev$IvC>OAj`G@ajC%V`e zj}rT7iXj8wQQ?wnyUq}xN(ZU3X=k7g_7D*It=zz((tyr58p2L%R(NQ{XPO|p|LWrP zyq2QeyI6ggu~0h&CqL@5Oz&BJX`jB&;oM0i?Ddn198+^HtJmjk3E2@IFdZVC)%cy$ zmTvg(`@6xJXS1(Zm`OG0mgZ^zZT zOu_7?ww#g^@P6PG6T{nvCht(j{Yxsw+-(f9^F)5PW$sL|!$wjklLU%CS^So1G_Ogt zyOOFMyjZK&b%#(gJdv4a1vqJX?q7TdXOHugtG`$@hNDgwE%nXcU-zJjX$5 z8D1FFmo20*BBLByo)K)5cXw+b>ObDU$3u36Yn?D}h;ebbxu+cW{#6CGXL|ll%?K&e zIs&1G47DQ4yB=kMYCu5EIvQqUemb!% z56vA;kN)AGh{LZ<@ZawLNj(eOYoVn4dk)npD4qp{8h>=Tyv;;xfQ4Eu_*CtazapdR z%w~a-5JqQ0dqKdE@OrHqF;j@$5>fO-NKkdq!C9q?g^|qY7&ouBjp$`2z0B4IFsQ>n zlG9ydf0=f#%$HaVDRF`{l^R3G$ONicGoU})5{Ez6SHp_1Hb;)gSbu5j{d9@;zsFXw zp_Zb`wJDjAJ>RoM``C##?SoI4YtzNsVDhoGpOL*;--Zp~c)ju@ohVDuea@Gs+@5$% z@^*MBUEFna@ExNqZk7Ty;|e{1NF(X*a7DQy1q~`356v0%l!V`kko&54*+NK-H{T&N zN(1tKerHuy8YOT}F3eEVGOxuc5HLUDBJ?KW$&jJ6Z->hz{X|5e z(d=L^2xl!C!{2H@a9T{+1t+bU0Ol92M|{8T=Ru-9p}NA#wIm;sx^;*wmG9VNjk^F8 zd9K*yv$w1XI4(fz>FBn5<(byPad@K9hSB^ai6}~hIe^&_6)~((pz~9mq_c+)7Q4To_hku9cRMcbG0I52>@RqghvL8eYYfG#;^Hdnq5~sNj0X+s@%8>BZZB2C0xC9B49TMI)Yf2q)S*Kr!P}tV$3;aYSK}VC{)^is+ypCV z^y>t8&OuJEc(Wix-rx;ivhOqAR^u_ge6J8GR7XuvaAvZe7ht&HZsHN~zJB!J@sCB% zGcan#f2%fp%Sf+abWm~6va1D_&&TwTD?31x<96bepJVW1!y$A8e9U>aV|Lv{LZRp~ z#HL_KKxbnXhfOo*keU~Nky4|6JZb_b4ktb_sXu*c-YglAYWu6hG>8MX8W$#v`H$|N znsl0z?kTCgmq@s*NK*yok&#>FY)HF^*4Z7h1W%}y2EGNlwOK`S%7tYu`Tm_#F?-E; zt;%;~$2q)KE`3Iqp*Fgkb5~;AwD4Yh^{AQHR1>%gO!=A=$5Bp6d2ExebS&HSM?4u- zPo6d#k0-JiF$IA`ZNJUd>EgH#FiK;|Z~0BwRu{;%pMUY+FYdfQv+V;~ec#*l)h|(@ zEhP|M*rf`eGTL9!0+fyO)@#+vSUQ=oa@z{H_2Q)+h<#FE>ck=VhG4psdUNachan}G zp#~OLzZ)D0(F07c-I~Djq!KZ7gGr_HZ`{W?nig|;`;y^_#f6@nUE|C%!Cceiw{-EO zD(U{`I!?@yLu~dJ}6RI96<573w zC>A3__a3jbDb8{V4aH$0j8B}VPTv@(j!ynXR9k5L{r$V{`DyFVhDP?y$5sxA1mCgb znOYKna*OYudGR|{^FmeBr3F3t+Ey#2a9k-(qXw8rY&>no=haMK7Z6mvBv2*hNE(Kt zoK86ja9lrQPmm8B*e7rWJ>&@kJ3)QYS%L!8BE$r;u5YkY5i*IG z-;o4ycf9=w1Sw~{EK}|>;n#nghshaYWVHAtwf~Xyki=7wqEHq5Q*Vya#Rg7thDxmr z6f}tn9to%LLb9j??KQ#&L9yn=Q-i&r5#+;xu|_?iA#(&XsC^Tx78cnH?8O0Vg}DM_ z{8$2NRh2mwnN`n<4UCOP4(VD4YJv$v1s<&chNdR7bKk9o=&cm?G6g!LK^UbG^g=zN zK^UYN)WypzU^Z>j_*T8&c{7L$~8Hw%ob!nmYVaOY1Y|91$lDAEiOQ%mdr_B-Xh80hVd!h$l2@` zLzM$tln~4oL*?^pl<4&qL#2bp%0Yq?f7scm{{=JQM#5U~_e&KIU>drMumCEC(4!Wg zi3HWR5}_6QZPUYaqtuCp2HV~V#&sis%r*e@dtg=q`QW`TF2pc~MAE0elDL`82n}l< zLr~Jt@x~3Q@Fmcf_hn9Fmy`z%If4w%11)2S0N$DzVtCY_GCWr%07)EoR#Z{NHINYp zR+kN=nTnv^Vv`AlX(1|@7ANWtGe$8LMt&(c58BqAkXyd&Xr4v-yFUdEU+q7lu>8Hh z;zllUNN90^eaJm@p@x6kufvPEfP@wiY0+ex5PeE9m8jD9uo!D$<-j##Z+hJ~RmV{`wSiS|R;fPf7-F+&kAf+4g9ypePXuw z8!cpa4aA4C(kv=qo4J>pn4=l?V!Bia*IjQ!=u-41Im^)HDfti@_ZR<61C?g}bnS6g z>2)UcoZ#)ZJbhF=g{my8EXFpQD%W`))PB8pIu6Lfv74+=n z`pozkI!)UWvNiW5@5ghVa&D#Dh zA~~N_EZUbFd+&6$lK@US>2+XUIiJi|@27DyY}w_}7)^OqrN$sn5bB1wu_5vfR0(|V zJp|O}llzJM{z~At+DEgr1(u@_6L;jev^-8pG(W2iQ4_^N-sGUUbVrz}P`BaGxt*VV zB0;BB=PkiD!zrccMyDPawv;_^H;?RpvnU~+RmgU18&owJnakeqD{XSu1*vF~??7Q1 z!UfTLHwiDez4cYfn?8|b)WHD_SBMk$nDHu%8{Gb4M%g?3RRKoTc3uLBta z>EYf?jtht3Hqc@Hz=48ZhvZ}x*X7gcr4~lvC&rUXV0lp7dW@TEAp>U;p&~Tsl0A!G zskE%G26xNRpOqR)lo6}xwY^tp&0p@MOHq8*XDJk}&N}o`=_}ZeVJWyBA3z?KDoAIo zK#PifB{_iIQ#qj$;3DPt^1`t!K-$vJmOU^3a&xD9_DQ5Ej0;S6U))E9i%@miyF61? zPnuY&>5Hhr2OW++l{tRS!SK!)&$x)s!Kpu6O)Pb9Ft2HM91TMeQOu^H)u&QSY{u!A zA-0*;N&cYw`eyZ6tl}|J>aj5=U5W=1wO*(a%UJvlJ;|bGmi{wQtzQ5P_ zJk4?#sKOc!VXsPbe7GmU@d6+!?BMY3`G2q-<{Hb>7LpUOcsuFB!}T_4SCp< zQSjMFhPm`kdojVR0ITKdAn@*@m0mcM{`n8+T80H;vP}{oq7vchXGwM3ZjGguL1Q-n zv5532CTt>ZrNvE&^?yF1vaIbf!uvG}l1?wSv;yk)O{ebU&Wf*Z#{=JLv1osHdaFZ8gGF?$|laSi>+V`2Cg+bgUCeB z89C@GlEdJ?kFbxItl?B`6QvY058vZHG@puklRGY>4!jud*DsHmwIrPkpR z1nSS7TAf?McI5F307akE(cDL#{N?5oW;X0`RIIjm%q&GU%h$uYohlN!O^TR`7!(`A$|_+ zjQT{sU^wcS+VA2xVp>k=em~N5(mv|UW32RH2^i;ui$4VU_HTR)`gJ-8isDxB*;hR` zDQ!BPd-V)58)7N(B61v&3q47Y9v={qoCqjz-gf1faT{}H-BPQ2QIq2DXf3!`N%hCs zVvA?8ND^MJfK*naYh?l$#r1f)O~&0%CFE(4PbXBOw}CiWQS4NzfY%W)K6z(^Wh6Fa zKkq_m{IK(v>sb>h{ayQ>XiP)<)kUCpiQW?z}esjwa5D|h|0$GT%dPBCbT#12$T7qf3( zF?&sO3Pu~Pzmx$~0SAN2lkDGC*b{DWkPT!ce>VbRTNQJ8h2itYox6=xX(KM-0Po16 zWfw%GPj-$OgGhY`_Cz=>-S2hgzlIb44ow-h3k0uBVvC_>-OqJX9M!VJGdJ&&t1<>4 zmHQe@gQ>!gB=}$?SglSsrN`q!p&_LW+l08Kv(oYcwDEqK((4)grq2Q6Ll_haml4grxk#bnMEX-J9v(RqKhAtdu(`u`9CZlq zi-$5UAZ`}&3^5aft_wgAei!Y`L)vzWtE(55%&iAjCh-!(q>!dI6{A@Gj31PSv>z*0 zp>1zzAHTXxOvUMxng&_{cPm(Y_XH;#C2&%fu^Jn_Wo|rq!Mb_JmSw)y;!I$eJUtiD zD2ew=(X=D-ddhIE@ze-4f1-)Q#?&$R%Xqx?uqIqMGGA27RT$Vji*bre7839tRRrmv zqdI3&*1&&@X^{SzDJbh|D&=LA1Xi1!)H#avikNZZJACArtl}$9U-bKvP-~0h^h;o} zMBte|Cw4W|4u5^-vcJyWTAJUW-BVHw9;(q@O3yv+bB-oJjNCH0>8~VxBM&NW*pZyH z_;$E8JG=An^eFm4IQvd-+ml-7TW@?jPGWK(U?`Aw5#o=neNe40Jzkz!7*0^GApymM z8$#p;zBXrD9Znb2xhhI2%bfTfopCu7Nl`3)LRQKdXbxE`p|K1_)!4qzH%Z zQ(du2D&86mc%&!gS_{x%OeSy_6j1|pY#sFinR-1(1smoXwLU}-PE(m{gcR#at=rArhh-mul@*INSfy~w(<*{)>ZI#-q zlNj6Q9uKo$Aw7NNu9PJ=_H9_?XPVLgUgd_`V!G*%ZiHXj@C?o6KN7Ohrd5Yl^s1a7 z5y>?m0z25M;imU_>v&Ca=f06Qrbv6Nt&CiN&5Y*8EiFV?q?f%Gm>$eCCsgPFqaCVI z99;Sls^y!Q2mtnkbDc@%p%ebNXqZyVM@rMEtci%9Q?c|_g~l_m_J=-2h6W>cl2_i; z^24#nSB3{r6Cg?9Zl5N(z{E)js|!nuV@26%bwzE08WybpTTH5j>UGGk{%w(pKMOud2useg|UQKw_NYEin>% z&0S#{R{I6vd(SkQ@!q=`2^IMZZu-a~NH#DVyO0M#n{}HGFSx`Vi)@M+E1eXeaNC*| zDm;njpHx;yK>F76)<&~)4@dSeAWF}L2=QA06E)?`q)y~J^Y3aeVUnu~JA(-gEJYyjcIsWnjAwlnc+ z$Fy{tY4;BFY7Y~2fEKz+l7=IIOhD~WMvMYz>ad|AADUo0g4%F1B^M;GGsvs)my-nz z1MM=f)wMx$p%_Eep7?f}uczZxZ!m{ydV2^j^DB|IHR9oUA4IVuC+5S<+NDkhQ#Jxe zd&EH#J_uW!N)2!ftNL(##~z}1)bSgep?dD4ZoH0`8?4GG^DCYQA0a?aHN|L05bR#e7sa62q?BO0e$ny65^dZkqNrn;{!-wUWFpr{X< zndWU%g4qh_;cgUobIvU?F{7EO88dt~-E7=zTdx0XEC{Uz`1U9q?0f<79B5Hc+wdn= z3X=`j5bfOPp({zoz|l@~@`mRx@%|Qo$r-$zd^Y(>MQ_WM@4yqu%{=N`4-ssCx30{0#p-)Bk=( zekMN?az(C)`>*e{fi9eq2e7QJ=p9;bFCc+PHTLAQ$zJH$o=hH9-@{?v8SCi&HiIJR?^kwf1ikIPSrJBgy-tHotg( z1Rfo$&}hK!Cac<4+uCf4f|Y6#5eYj*32jTR+!~+GI6z`E)csh1GpgJe5x+AI7w(cm z%&m6miQM5YQ*xn!jKLGjOa?FI!s6Dvv&D; zQc3KV&3Sfa63C3BWX6^h%FaWHHFsa=A@?j(b=J688KCFrZ7SQ-^h9qExH077gm!+{ z<&V~!)smY)iuR>>dk^!TL;Km8GYYiho?b;dVmx&Od>t`FtNLmL0}%45wk?OmOJ zIB9x&3XuVZNer8%Eb~ypo#PpvS^-7!ss;vfMsoH5r=k~NaA44yNN0PPo+-J3){>cO zgu_fvs);q4+-E1BkqLQns#bZ9+XyDn6-{)&w@(ePmP#p5Ga{jg0Fdtp4{gQHQQ(FT z$qmKM&Pk6_*HYem+jCo`Za-&oE>dnoA*Brc)}fZE#6IcM}zJdg*5!wySt zu>V)!b9NhXuKuuV3(hHf)M=^J962G8T*5SeU0|j+E!lH2A%7FOD4(O$n*(&UJ<0}~ z;k8V;fa#rsSwhS^SSOxuOWoQwvP`!({L=TsXCnAA>_O`uH*m&qG@!PFlhm&;Y#xDH zt$=A~KTR@r&oa9aOsxoKk^45WT49%}o05whmzZyx%`>_f+$%u~hZ@-%2yz&^3EZcA zTL4}$%m*-N%e&lrm$h&4ILjSZj^gg3uJqf^0I_GG4@C79OYugpArcKO8#XyC{L;kb zoVC{Jur;E+LCiozEd61U@AGY!`S7+u7?^F=`LHPpqpm*cIyY$Z8pkOB!@ANVJT#l(-k`!Rv2Ma)yl)2XsnL91hq3dd3pv(5X}pH8pDx&vfUyj zP9B}%TYu6>XUmqknUt+_uaO&qohImzZ{7n%q)A zmxJ@n2Wh_G_Wr88(dA2NLhSg!nS8z0%)$XkxubmZxZv!bRR9p(RqPFodV;|QEwCYf`TB*iJ3V@XYE&~MCPYHQb%L;@FO_CfxA4Jg$ie%WSQWSInt2L@` z-G`b`e-y}8dN-5+LkD;H;)P6Dl#-&=0(FK;2McJvOxJ{#qRw>ygh!0mIZbam4Snli z{%XD0CR8ci*cUVKbo%tk*2A28paDARyIcctn>-6f-s;?bv&Oe>g_0V+&a*gW`i;)y zP!COv%vF^&JHxLYN?x&uw6&8t%_J5AZpylzw(}6-7Js*9LS$?}Qk3;<>wT#=AS}XW zTiPXfJS_OMwr*>Sx?MXuA}4s~u_UDs)#=vLr@Q&;LlKww15mn(=@&759+q*MvGDZy z3+?oie8n?bW^12(ca1#(DT(Fxyc8g_lY3gCylJatO7gEK-(I}?;qAqnSEq`jc!Y4> z+a?d)ZMpNB?gR}l-Oj|BcB>fx-emfrINhpN!f3W8>;ss`?zi2i0U^ztLcPB#KC?&3 z3#(8@NkCD;k0#hJs7K2^U6IEXaQf&Pk;lCK7Z6_BDK~1CQm0BA)rbv{UDt05P+f#t zpy-NFy1oO+ksCX}HgXMpkgqZ-E1rCjiR4c8J<>4K_hhT5Il1`mn!M!lE?hYiD`vJ) zDyS(lrj#HM4-BrRjV@sy3L*U_S2H%-)@8NDSRW+A?#@(qq2A??A{LXLHkUU6h{^VF zAcPil-fUIVA)iLd*n21$TiWPV8S74L)aId1$b->8) z(+bGO8nus@W$Jv<5T?=@~QuSBgK4tKCm3$e-f}O<#VUVam3=*MDkd4J*o-oh|WvzYAb5SPDz#PLZ-qDe)st^FNWybiZ z44y=3Cd5inmoMYT$vlDRp8Us}k;|8c!8-uUDVKNNfwDE0IiAEP7c2Uc$jPfOu646c zf1yG1a-L9n<;0GuSHkw>s?ne7k}0_)O3<-)cZD1EJ$XqhiZTvxAfFZ59*9bOnF0I) zJmLr`JIE6Wu>Hjwk&X~5oQDj{{+nvy3;Z7H;XfyDMb7*w-0v7oiQ2hm`0=}vSIotqz$7OlRIYz=)}uDRqNBi~NU0CT6A>);@ai&AZD{o%rV6rRjtEDCQf@ zhbH*+7N@gh*y6qdYNU=V?CE5XqOP+bUl$EyApIeq_D`t6i?6~ncK(cJ>?ml@r+k%B_EtljdX}O%%I!@2;Uun$vikH6zwR#gwJBbalJQu^X_t|vs!VI__JEw-FsHU57(@` z=Lria4B8WSxZ`^J#gzV97n2g*laf`VPW0vxa~@i>WBjR&N#yM`t^#e|BNv(%0c|zM zzF@0J76F`QT>s1yrhCW`;cMdASdmQ%Z{TKthi@@$+;$TutXR(DGU?QD>OV@WbTXCBBssnNH&EWRns$y8aQ^Pq#mhIZrmOU~OYtzN#H5yN!Ykj&7y-2t z3H}lTP*Zpud8EhOR4vsIVthHxo|A7Rn)AU9W?Yy;RDSKHq~Ia2#Y^XPmf=`eC1Yo< zrRn_ROA(FXq}A1cOc7Mqfa1Q70rGJc<(SjPCp&D8b@NPnw69y=hqOmnn3aHtaX`A7 zi&Cx4vb~mAmB1jJ*OHY}eD$X?xS@j%8vBlU%bNBL?K?K_8(ICe_5Au74oYCOu9g^y z50+Og4)hK+0+#*qV;hTG=!+QXA6?hdp+~VAZQT?1N;qZ@=MSR1N-#3Pk3#riwo%S+AYunI${A1G zDAQc1u}Cb&<-7B|k&ht#eUf^A)VaCCaK_1a+sh#hUoHe>b82kcpeUxx`DwcYy9x{gS)!7xvp? zaN!$T#95Ne-5puLevJ@T7|dFT0)Qx}inb1K6al|8f2G+XfV+{Um6Ob~sxh!v1Fd z@$=88htEjH?--fOzm}pd8YE&2hxO9buyojnW-UrqeZ)AVpnbBnm*DP> zM5yzm8;sq8^3n%KOLQy;gnnQKN&8?;SV~@4He-U6i&9|KProBMHx=`=vLdz6ZBpJH zSAhW}P(nxu&{{vDi$!f#RLQths}qpXwrp_|m#tR5#I;oC8 zn|mz}u?if^JqhI^z|4M&lKCQsWi!@Zr_2#jD+X*nQ1X zku%Qiphb)bl!GiBCNd>&G*iaSMQ2iEb%n%Z7Q>r1Pj93?Q#b;cwY?3vcm&x7{Q;SB zh~?|6JcP4=7GOt)UrwKcCEexz!vK1>m zaHl31(&W4?X`X73!Gx8-NZ5ypm1-`UV*iqyeBnz{*Pv^Ip8qKG1iUFeP#JDFtl2F> z-!vP0iCqk-sMf+VRqGAXF@Y7XMO|v2mnkm-1p5l1ASe+cWalv5 z?Lwm)m&kk4BPk1=!zt$u%`MO zz#a=TA`rTPQiZKSH`<9J9dL4y8v;;B-Mptk2JpAGWO@VmmffSX~T?Ob9sNdIZ5y&-4bq6*!(F2?~+bqIDZ8z>FV{C*1 zkO%Nf4?+txSR}Pl#E>7WIcwq-!lgSSmTp9X1hjKou{ISC`2#J}i>tx>v9?Io-0upd z=4+z9r!xRn#vBHZ!)wz!_Dg!f`V4H9P^@gjljU6!yM+OPBph4dW+u}z$w3#Zmi$pWy9%EyzQoaIg)rtk`a2-HnZr+_6B6vUvYzs z@_+!XfCAIg=i?@Y1mR%OIp}yG3lT8u;RV7D)Iz~J3}?0wX>PKiK(Z`o!o0X!&1DbP z2=j9MKoO{;Zp71=y9ba#_4vqS$yO9TeyPT7086Xy;*uy~u3`vg!pfB-Vt&?&%R9P% z8qnR%HL5bTUNAnGyY{sSdS^}LYOypGVk1)Fo2zla!nVedJwSKk4Aca~t9kS+Mk*qB zZKsalwT&AwAb+?rM%4Ch+=pk@65DnkwnU4VT^JTVx&lTF)hH39EoMZJjd343&cPVx zQH1-@ku0K|voY>NN4|(+6HyXKOUj5K`>S>zI)~_oodq zui9$blNhsC%a;4__;@-!#=wt8VW_M5|(7xVUU{L8+O#ELmP<>jTPMcH~0I?9J7ll*U~K=2Hj4-h!<~g@q8u zl?l(X{GQy?Qigj>#*0}gzvog!Ef;knuD_00tOOzSM=-heGtrIjOu@Caa5~tk$eBY$ z3EP(j*=+VkN9KL%hsH5iL92?D`E&ALC+~m%^W^j|Cnw)Onf&|Te}D3R3Lk!V`j^w6 z%^$x%Jv}-3{;M~Cy}5ezcYgZw_j$d#!@qt$`JTP{dnA$5)4%*{RP$OAAJ40aC?`1d z&jG70f`}QQZT3rZNLeOkrSt zaW#B7P3}-jvcowAxyi}1Jv!2U@D-yWTOgXEArL~OOpUi8gl%e>f6;{+u~(boQlcbp zKukFJ^pR)~0R9tutz;apGb(gS#-EMRCt@ez&ot9Ss#F5>ObsV9hq_r=1Y90zOEIu` zUYWgW9D!;HsW+nZ2WHVs`VHQ+C^DV^^gU>*FwJ(^+yJKU2dE=oOSQ;FuJ7?u)jP&E z@WG(=12~E9-DzH1`$teOl&&_f z>nh?Kyq(WPzHsrjnjNZ+gB%Kc-t)Ak86E`K_(75{t}l@+Uo5N10|U=4p6Ly*F^%Ni z0M+4LCgGr|w4D#F@lX_#t^qes$3wP4Oah9vUXgt(0J?c$D<&#o5{%+%YNfAe>t-LJ zykr9uZokGO5i1WXJzz+0kCET+oAqJSU?U2`@j(BS)Q%Qipz@Lu_?J$0*4#AfHRpB!02Zwsq>2a zqmR^+_Wdvnu8?v$C$dh|p;(k`V#J});_qXq8Z@X7I*MJrX-P4%t4M>L$D+@<-jdx| zZs@znlSUMRi46eSyafp;TkEJHv=(T;PR^ zNv;!^DN1IgxMR6ZO;D%7jZX+9CMp1D#ze3}=!PK4lI9E6q;jul3Txyms{w?p8D=)< zTuIoxf@zs4fy8>e-scw66ur#;y{d_ED_$~#Yk)<>VW8_A7j}Ed!-i?avlpzAO>icj zXNoau=|x*^;IM;f2vVimSkbQ-;3P_+&n0#GhB`mSMThm^-iS2LpVukf);!!KUtn^N z1oIZ&5ao7Q0hOv-&6X_b{q~mTMtYstN;A-;3A}FdG{z+o5m9Kak)F1gd{uKP9(5h< z7V04X*k?7X*Kr#*GAN=vmAM#KEa*A;T}!~x!m!ooF(K!~sQ6PoAqG3DeS3_IF#P8xwh1}U$yB+QEkrN7JWQ?Tg0a}kTr_1mh2t8NS)5s#KXFFs<;q_ zYNJ&y5>9VVA|g7*#&kd=EyRPx-QD3v7gv|?u+a+_@P|3=qn*U&nlZ%=8pI?RX!jkX zL-g=!@8PeYZijZf*<$P^Q*VkU2|b-cnLwJab&y`Q_zvpndL+EGA2jUT&-TZr8kVlXZSr_aN>TPF)4_j#6i;`UA z6P>JYoi|$UHW9>VQ=oaAWT%BTXst{B;_6Z}>62XK$oBBwIKd4qX}!t~lCjc<$tRCJ zVI5c|BOdWQRp$X5i(tr*b&{~it)$I{ZYAXA2++f`qE)hFl1LG?+zH{itZ0@swYv2J z5Fl}CUy>_`#<6fG`(9f>i;QX$)x6FV40YtydQ6bshN1vg4T)AdK1WYNV_k@2sb(R8 zWp0h9=+ac41w~n-34ST!A{u8jxqF~xDgoetRvas~ouSm`3RwB^aoiSdrkUqmIn(X* zN%LFWHtou&r&jir%+>&F1|g1UU5OQ~cmftcK6kupD_f<)#8Sj?ToEm2g;%3Q?6G9217`|lU}mO&AC!a9fdNubAt(CU8YWhHzqPp zO86c54t(eiB{Z*QnRt+-dJxv?h0w4l4l)tK+{#?rJBklR7c!W1afC;uRpLX zQ~&xncEI%Q={;ZMnBC8%F-_{^nv=8?MG-3~skMIE?LCBnl=nB!t*x;2K)A~Qoy=xRM{;Us+BF3*C#n@-HgDu~0Wj1l`9_z3x95w^ z%O0=~#0f!YLqPigdTz9dU`kFdvSR7HIr3n!yve4nwnxQncNE1JrI7NiAvkUlaib@A6fMiX{I2$P0(9$UI z&Ih`l=B(0qRK_Rp&b;@pB@KX(yO&nWoQ-F=fFCwK)$^p+F{k_cDO(*-$U0npfAilz<>ElPIJMIAQ|I*p;OP{NRv*43AMVJo|uP-O_cDbsju7({zm0hAjD1lQY`a9+FYX2Jd-|8SVJx zOsj<^vxxfZ;dd`F2OC~pF(+kS^Muh`qJ_G6zM?r@u!$u*X_AAy)tB9Cq_cw9RQ$*N z@z1+I-WiyO&JMwoIev0&Ug72rX@TUL_l=Rs#fn!IGhxnlXFz;P8i}vxUYaL>0|n!n z*#}!|(@_9G(|+KdER61@MD3jLRe_}tU~f-!4xVuwj2?jmA~gy7kT90&#{c@*&8XWE z0w4~PqP3pQgBqES&R9l%tXa7xN-i{6A+a$$iO3~Sb?%Dx*2QQ7W5pD3)4lqjWMe?P z+DgCf5@~}I!casnIXjH5#EQY9Eg3e}t){Mvw=d)Ms1ad|WW!RZ=e@5J(G&>|pQQnu z!iJ4nYdyJR>v-cJ(Sg^xRInk783(QMW+O<3A${A|*u{xgRQW3R8Y*Rla|5+#;tB=1 z7ecJ-DS-@fHEbfj^JE*l7hY!0b{Gab?X4u>JGKrWI&WTxY9(^H37jUP2zL zq>k@72tuzR(dpx8wp+~UrsReVMyX>(3&nN*uM#d`1O2}ddMq#Hlw1(W^F_v78pf>W zye^uoYLU{DeysUD%^2o0(%ja36Y~xEGoD*ukyj-W*xbIOJGMSL>++>&ibt3EQG@^4 z;z3r1gP4XR_$?B?nCMKct_VC%v^}KWv}*{ew$r}ID7sU|+xoi{dJ1JljnlP5r*W3) z%-IbibQH-Q4b=d-5toEDiXn7MitI4>gQT2{-RJ}s2zOgDoh{WN>LfBpZc9C9n?;!2 z`ddx1Q4~#4>V)Zy@MNg})l_K-*W+uhCQPGbvJe=e+dBv&B$y?Xv^HUT6)Ebn5K`BD z>h761&dDvJu}KfbtYy1FAca(?Ek{wjXk)Z#2NM8KY2UKwABZUGCX^J_He2fkn~I6$ z$Hy3VqiyQ3D^;mdl^X+AiFQXI9X?#y+FX)h5pJhmx;;zAfyuZR;Bn1Fo$GaPP`-8_ zQPq2cDPWtYcSDmH7}vrDmiE}qw_j9hO*QA2sOv9VbaPInJILBni z=?Gm)t{84xS<#A}VLiloX3C~SA3`gU^0|&6ymp+SX9y;t@gQ6f<|wjec=zW-#eW-y zmqgD*H=40Oj&l1o>WQR0&`BDquw<^Dx?X^8ZvfC*P@53|GUnog^2)csPsN;k8COQq zp~zRrG%;yFRD|bAR;L(0ge+9?su(`M#gJ|#jWmxM8PMdr*wCU_2H(ecA+f@)*(45U zVN~xUCGQv|djhlLxgp|R!CU5oAQtU{tV!Ga_$8l}jJ#OVJZG5$#Eh=7!8X&XqRA4^ z0!kF$%2;(Q$x_IQtZ1_2xrNBgEu3~Bf1(hBqDqX}dSq$ZV_ln8FVCj9D{lLyWbK;} z%nOuL>ptI9-P3FUh6dGHIGo^K|Dqi(mP;;I)@R9c=Uis`0WV$Nl5VEDSmshZkW`Qd z27yRQPKC?@8B;OG={+RnC1O>SV#Q6Ayjc@F1Mp_Gwa(%V zE9pJ70?ep_;AhG2iyp8hpfH5eb_?U7EjEC1=R)3`C(_WTym%#7F$-e5>Uz8 zLU95!);mkKfiBQM$>${Z2QIc-*UIKK_PO2;__zKFG2Cem>G(+C3=)y?v+Su#j=k|W zR`=9fCLnUp$K=$tSH@fC@G<#RqpfV>o|X5UJ)G-|ugL?imJ^+K8kuwdvtt4~t{WoKTXjq4unR^D@y$VtjN&xa>1N;IPtm8=SxeskM+-h&3;t8~wyfc%%ulYo_$l-uG(ozAVaK+O>&> zw2Z^pOAA_7pl-!4PWy;qh0YiGuB-EMZW4OhesC`BO@1ZYV#uje$X$V7+PW;{JZ1Oi zGNn(?pb0VAy6t{aNsgW#P02N1@r;%k0)kmoG;^QdBrZzZ5D02}azsu<2`x%!PBLbI z#WaUmqgD{m**+HABYQW07wIU`I)$`lrp6ip~y^}RZ)&LWs^QrkBkEEcXI2YDVLqv(i@^q>B>GXICIT^>(VvW8C^p?S(fbX+E~M!cpCvusLO_Pim)TflBr&)ml9u+0U(*I14_ z+oGH1L{G7^IgJ|l?)C2Nz}+~a6}rTiZbKqgE38Jdw2)_U0GF0??AC`cXAMq$n7d*X zp!wjI?eJG~dq+dHU+a`~f9L|JHNdY%RgTsrwxq#>(ns34ZPfX@R~Ikeyqd1kU04*$ z(?amv57E%6qir5BYsY-yRPxgJOMaQjj$ir4dxL8@mmZ=p3sRJ;oa4mYB@%u zsg`Cynx0u{zFJ0S6a3;G-?LEI9gUEGW5#e}L4Ck#=~Uq=Sh?bu`_qyZ{W{X1Bzd?2 z-c^XSTWna&9InD2WP^$$U3nIFdv49$Z6;WhY+|d=%z4HDz1y6%!@k?RG4&YBoU!BK z)7iQc%IC;pgJi*k&ZWkz&9Ze<2vA!`=mIOll-g5Ll>DA&Y~eEmE_uFiP_$fZE{#{$ zHsrrie>FH4gM?v!y#4wH3ipD)cxOdNHN>+d2$@VKU{X5yhmx7o?38@R^!p*bp(=S& zk%y(QWm2JU;TeUkiuK;~)8~qUl0WfyY|-KB{KLX}=sn&^zTSdE8*96j3Y`@>jVYC6 z_&gAoTR%S3Lt9|x2!zOgFhpWsV z9YOB`68kUQ^9a>`2cFAiB6%FcgUu5Voh2HE( zR!x?)xTve;OD+L~F}K zInJYdsGr-Re}=*#dbi*3`Ia9mV0ow27Je*0P2IB5ek{NL>LY~W?|9O{j{fiByW%;o zIIToE%5Ttm%9>DWS&0?yFfe2CAa2E5Fy>v(HwiX*p`Zi7m|x~fD+vKNx8tABNX{N~ zo49$N{D(FxLbZ?V&(EXFmp7#fO~qGjD@sJ3vm{hBH*QLPjAwiN$Gl@H#jgI(nr1kB zZ}(y5YY%^X>spVQhBnji*4j&I8Ho0xx$aq2(p;)}Yx_%%wUhIhoK#8S_3!OH2PUg? zS6sn%-r#h+wmC4qChge?2&+2YqNVS*^W(Pv3VtK6xrET1P(6CMnM++379_VTrF8P* z2b!kkb3B9RT9PV*3FD>|1I?`$`=nYIy4KOniz}ca$-n*a2}p9EJ^SMm@8o~nypz$E zq>t)d+*zC%A6wjtAqH#;C1qV>c&*~4dXvNA3owMVIW^+tw8z-kUwLe-NpC~O6ut1= zhala1lgVMG2Y&W%tx^2V;bC5?`{8eUQS|wbLrtBi!*1|5caXO=6!5YP`aMFT~()iJ~#pgvUfW8(=8=c4GI*8E2oFF&3QUZej#_1>N$=^$a6H_5fv6J zb`GwK8qu)eQjW>MJjVNt%%b)WGM`ehf5rHjlrh+A-BtICuB@F5z7yS#ZP|R$2SpUe z;d&p{5JCn;so{T~ZKX5V44_dTHj;^>p)04F*6{k?AKp+GW(wd}c4D4DVO(^meWaUw zJTP1#g=BLm36Y)*6uEIKwOvX(?9qRwwEBIT^mpB=MLHsp zP2Ft9{PLxiFij?!)QgNntTM8>(hh#2rvCxS-*)=eTE)|_JuTyVtXb<`&Vym={+=6~Ti1oX5w&16=^|n7H1E>u4*!1>hD^!M^^$%R~BY2K^ zKpQgD#5Ej;g3-#^ONZtnrZro0l2ZHwphDxAh`Ae$_gJJV%harh+%1Epffa_s?Xg7D ziWXP~o}}U-f1qXRDHa`-=70+^1dG{DbZDM|;5zW>sy}AD(%8=$A==Tk{jQR{aY`yS zqF9E0i(?ndiAXA%nbMZ5dSJ{iO+I<@1Z|H`{`99m5hxIrvV^a?LI(qt!B4VF}=p z=W_?zmP^AK4B!Y~EKLD&wZ<|YK#~V@3fhGY<_(5f!h7sV0$n&G8NXv>E}`_eKbOQ{ zD3?qskzb=%A^JNiKwdq{#Cz=ylS7HJ%^d{JPO5W!LaFx#h_c#W~ zTuV@!?Qv$`#Zr#Jx_zJb(6g;q+fO@+W~QoEY*TZ(VrPom8i35b2 z7GNU=PYHP?MFog{in5*{X@S8Zm}D}`MB;}Nz$+0eT~j1a2|PrWjHaggv=WPof~8YJ zUZ~AD=LxN%iW#b>QPvR`6Fv$e-sa|#V~W~A!5loWyNjQoe}qcH z;LMGr$j#QXacZoZDh;lxX_l=g?1N&@{GP=e7Usf~R$UK)RcE-Ui;`7{+jE5d$OU9% z9Ko%Tq@i#@T_Nhh5Mg%D%C)8)MR$CG!)@ztj;X*IO_F%ZSN7E496@(mgvoMR~h!Lju?x2?#zMNQ5FRps1MUCtef<84=)^I+9E4^*G?5EVLi zsnNN^gl$u&^Kg|q4^^x45Y;*lSg-RS6*~`7vvY^4omYZcjcWza| zbE_Jj+l;{$bv$>irfuYQ7Y1&8{1~cJyR6w6Nkx8K zY4c7YQUrghi+B~^3E@|V>E>jd$Wyg^^W#|VOz$br=qxjOK7=}kA;%VOIt zEkZ*sfY$&%h5N#0&SM#kZCtPY@vSyLw%5h@_EtihYJOs+=rj6^S!yBQYjcIdiJPmn z&fbCMaj)G+P>RVx?J9+cF1szjK#I=48T9lZd#ahNPsJMNn6Ib%F`nIMpcF(onQNeZ zU68V;;gQr$x@Vm?MM_(C-u=&4yin>5(CxlG>I_5&`5Rh{bSc^A+&T_~&O}|A>{$Fg zVz>YSy%NUzwrAt?2-T#W$_k+e?bw$y!f3KoBG?3koX9n$1WqGazEqvHzOwGQGh)<; zvsj&g3GH{WF;-Fs8O2fr88dexfQ=1K?=|x^^gmXP+pJKXajBYn$JTPQ7qRDw>1&fZ zuC|8`^{8{_9b?85>S{p>zhoL*nHsyElcc`nNi4_WSeRE+G%LYQ_E{{bLvu-1)de;e zAO=)#nUq+NnXG7e$5Ja<#m>@|8a!*BD9xb@e(K#c#*(v@qP~i(4cy!8 zV~9~{6&n_v3JL38@p>3u?dVVkceHG2W}`g9&Xj;QS~$#eO5W%LZ{*^kN(Po9t1AnEy7kG4 z7j(=R5E04RkZexV(cRWZV*DYjA02-{jJ3Frv9?F5LwIdzNsP6wZ)pY$f=ER6?ju7@ z;58O-B9d$jECpU4&(vrhHDymM%v?*7kL;i*iQ>_q{1|Z?;p^fK|8ZuoQ44<;Yq$xn zzqq;_X-#mR9Iz)icl>HC3YTb6Z~~af%cfi3juC&NHTvz9r;Vn4sgBAfjwAZma3{19 zI%T0be4FB+@lGL~KMaiKfC?GV4}cwYuQqpoXHGwDRYhw|e5>&i23{t84$e42^ITLI z40DF2_)(5TV#A2tyzJ7+J{u)FZX0XWoDVw%t!Nr%lw`7GD>_=vQrDMiX0Eh&*+epv zij^x##C&}4hnul>bpJFm+OoN}4v?6euIx0lR}O$Q+AKlE&c-S~#6bhF7Bt1|?{V%v z(y(k7{b7f4w4gjPoMU%iG+w(|WO^(y#=0_UmVB0&Gb8qlRjz!N7-L--M{^IUZ5uSB zQr$+83|V4eu0~M|TVh=6$|%wvOH6dW+$PPa{JGH2waQqrJXs}C?m>Nfvw}?>GXKa?AY1`T5YSV`Q+aS%M+2B9H2Qb zM>bLk>5UD*wP*pGw6>n{I|lnVrh{7@ttKP;%bf-0MohM-{}Cu#-OH9F(eiSWF`n@i z*V$l-oiaJn&v1eSrb$tsX=2kATZwXgW(levM;>fVYcw^8HX=z~mMpKb^+%S>?fAuq zkTh3<zP3w2)aDY_$A1x0Pt+S!0e--?IFk+|$yF-_&<)EnKJkp3Cu>5j0^z<+PI{ z1e65D?myDT-A?@rR^=aR-!G#5jd!ULL8Za;GA5&Vzq&$9dmgQd} zRt+KCBdIQhSV2dyGJn9yA%u|-yA0n5S<|dWu z?UY=Qk|n(08YY3__qggb@Nq1^<&|XF9Hk@B!$Rxfyt zN&!$-Qew6yrKo5Xy@EEo=&maBO7)URiYT&GPf2;CfK9JsR);!R5;OHFCde|w_z z-$d_rb;dxku}Y9EK5jsrPII?atyetzoGe9M%IV8-KBY>Kr|=}2l#G5|k&IDUk*80{ zisy9|3sK;xEPwob#5RmqR<;B+a1|8zY_*hVR-vs!P3;5e18wxW-&@k=F-g}M8c zq}fN_7#8J&gnq2WY+c2b>d^PLq51x&{&J&y;9$F*vVyvNU{gJC;Gq~DToB-ZSPD0o!}H5tiVNIZ3;6ZF-8By78U%w6!%m%xMT}1t8&bU z-~;W&siBfYNmg}M@glRM7JBWj4_1C|2z{QnegW zFjCzd;y)5i|0TaqSd1*HXztpHdvEEcjc%CfbasUX@YQ}_LD?PY=iSi>$_>Qe15HPyS`hk}2R|zok!uLwpc=yH2Z)C`B+(tL}Zz+6(i>41uK@8B0T`gH&@dPcqG8}N|vMzEtN39}dDb&6xdCDZQ{Ha-lUXnS_KE~+A zx^H3}xZ_;6AKNKA)*Tz=XdT7pf`+*5oQtFM#bZzyTMQDu*q?OeQpAmtRL|W%}N>Vs@>+bm}t))R3VX;a}OsLvoQ0I|?4_gSRyzXVqib z;$08z1+CZvjX$&X{bu-}cQj2)CZ*w^dX;#)cyV<}{)(h_uTx%%3dPTtVnw+(P@!8x zxAmaulzYI&bmq#^A)3%!ln%sit@Ku{PyT|Xp*KXLUr%Z1@gh?rF*$ZQlVX68J8F_* zrWXz!8_7T<%OwXVpcBiYOZafvax`qj%pQf9Am zI?Mc?*7kz7kB~1jx*!bxI8V7$|B~zL*WgL1bFB+_M*(~bn*Z92Qe+Cd?^R@UqY8gJ z&k&xcrXZ+;AE)mK2CGw>euKeAGe&NfJh{7aoF}4G-{sC%(MFsR&qb{>-L0!Jh6Cpr zb7C36>{^c!{6?h|2@e(mhP41zuI3m^;TYQhus~10y>}q&#CHtksGd{U%)Y*Z`dNo_ z>hj!-&CZ>{brs6??{O8qbrSGQr|dEv<0mq?W_q%X*tw*r1=;BiN0P%yWi|?=?jGFR zqy@*p-|c}te233F5mtV@39)_VmX2c`ztip|;4OtH>dY5}<_z=B{s(;jE^a6%$0l9} zcN25^v%8{@uX?h%%&G!; zhC%N_PX_v>Hv`?|mIuZ&D(`$`F7{xyc{q>UPuI;c-)r{xh@5dZ8{A7XCR2Pf2Rt}k zL;2x>%x^#cYiEV;rz**|wMZfyZ6h|PIJybf?A<2B}B#%$-9aSdTbwsV{#D2_RcW6RI&`9#KRr!3}ixyxgc0OAq= zqO+67fN6Sjk~i~=BPjM{A_udJBgk4qz^I(!Xhs*8TM*6~j>tUNTQWghrs>H1(k64% zFWrnUe=x?FKFW}B+&t53o-w#lLGePzV>XcLl~6qJz}hj1Tc=?g0=`uYiW+@(y05Q0 zPE`A~5wx|Bbl8dd=sVqq?{uHhq`#aOFa78A;>i}arZL6=^8hhT%wj5J0Qa@~@8ar` zBqf8)VVX%S3f;l8(ZlVfs-;CE)6uG8t0GVb_tA;>t9Rn<#UZe)S2Ulf9Sm+f=T!BM zK=%-t61`XPXa&n3q_ovJ|FLE? z#1GfF6XF~wXkS70W<#NkR)vZhlyH~9X1WKtns?PdolZVuWW~!;l-ir)cEd{(B2Sc_ z$Vya=N*$kY%__Y_VEI*o&>RzuLbd`E7z*7tzTz2M(t9rI5-C@L9O<8q)LFR}et2(V znQm(S!)VH$IkT%F!pNKoW0;#jo&ODTcDf8){(&XNds(Qn_FHd?S$hk(PqZb+`)l}V zTe(rA9AE~gNhbLvWJkZ;7_!j*Q_GFk_n+}(efe^H%m6-TP9JIG1=RQC3n~L_@D0uB z0*wqO*H>S>IaMnKtVfqGyB=s)+QJvS-94?bGX$ef-;E~^FbsRh+%)G`s5Dj2$$8ZU ztia#^gN-8%3yC>B?^o&FZXF2+1vIVRj=}ogFoC(~`N2zb*e3MAW}B~RmjeU?4UYZ; zn_*^qo|25`jLb8-fRLsf4RVSxU1Y4RRc-a8P;X0^lnk7Dwy4I?QU)<65oO6_0hmPE6X*8mSd!bxg*#eX*=d^MSp*HmxsX28oj9#idv&i_QkT41BUc>Z+XI0RYrZNv6GLdC z`73rM?4s)QihPqxR;d$pmeKsqf1XDwi0X!@jGg-iIOiu>2tfyZL4F4pVXHu4|x!;)l@;h_cHbW z>-xIM0orQr|JU{P#7QG+CZA^6_H)0>q_tbL`!GeAVpVgWMT$k33jurM{F$);OvzhO zX-pIS3zuLA^@#x1dI0C+&Fkwa`A&)HH?OZzVl?qyOQEOcts3iNRr1Axl|BnY*hdqXqKdDWs0Tu*Ta$BxcY?gE zxtFfoH>5ZWw7OrN&<(jgsKQWXadyC<0{CHth&5D)_=66RPHY)I@2?xa5YI;{x=pW`t>0H$R;|gRpb9DysDv`Kz%0 z_cn2*)Q3XJFYYPNKxx{1HWf*|V!4xKx)AnD8q#XbHeZO0<_pj*i@U{nQm%_ioL8AV z|7~%%m?R>X70oM85$a$T$p}@`7uTC)oaNOI3Ef7resQrOSR+06Yo8L=#_Y176pwPb zdp*kKj|XIk=%ZZzD3?FV<&SJ<_FTOMlJ_2>!Plf{>a1jcDM$~iWC@p;D~E& zL2L>+$v)XrkQcTA7%Rq^oRTjgOBCE{73CRVTQv8HutzXLetyyF`S7}rSno+)-m|L) z6)kocUuBv4oFD-mh~h>+6L8p*S1(@v?Uk>$G*9J{-mx?FKw%p~M47U3>gp-@qPBwK zp**CzL=CWd*X7G!UNyO6>+8VT8#s53^j6@0 zBF8Q6upEC-!Z8?L4KasgY}uvI;pjgYqYlMw4#XAxpeHYq5t#T+$H>%RgFz68VyX?i zkpHB+d42u#vyd~{yG&L*AK3w(@3{ldcIp5gO(;FQ7;h1LA@ZseSqH`tWfA0nx{?(f z?Z>|Etc0v|bIOik@r<-M{ITrZ?6Va)WDvi!tXY%jWC=D$^ckU0)w6^^;qXZeC%R&n z8khjept#1>Gc~t z&S3ac{g^p&+;|9*WVNg|5A}DZoL%p^wBXgUo=p?6a*n8TjT~C7&u5vKov)}OtPZTI zaGrNkJbbIBJIS|&?gv81X?c1{aENko3HuLELbw3c{;(ix6hklL4z8( z?;V54(Vl(Q7B56aGbfteK2HjfO3aEz0mv36UGZFv&RQ4JJ?)uwG*FmSA}*tbJQQ!@ zi#1U#;R7ndhB&3;@?ll5-bI1yZnJ@d;%k>@VPo9HGt7{2URW~3jFcgWQJV$SL690NJP!3#6WK7zzvVgx7&QoB+x*)UEO7MmyWp_=2U3Ncju^< zrrCqys#!UT^u$vTGB8PKJe{U8n(ZQapt^=4RqAeCh#a+l4vsB# zKJq!a?R(2m$_v2b*=!r~^w}Ai)%e=mDXySt!y&%^`*d?`yrlpAtkVT9iKxL*spAKs z0IFiyRc8SYl5Oz<)Ai8u_SJR{138t9%E(pl4sLZ3`1ig(`;PK~)?BxaZ`+uyUtM|Z zUhUJuDHYhv{*0-;T{84gZ`lfe(%mLM?Y#}hu=gkR`o5ESyR^NI_LXs2feLoV);&`= z@b*m-_`((KRzfbQ5@!QP`MZWK8RdZHYeJK%f#GAM&Qn%qP+%U9%f3TaTMfWShO=9+ zQf)exPKgGL$ymiKaSO3&ZyOlQev!M}PB_a&KB-u_;yI)TgM`) zGCE_KBr94a8np~`Dp8*9Q#z~jUByy|LVvegogAFuy><%Zm~_hy?QtsMh+Wcq11%vH zTd}&oI+XU;?S8H`*;G3^zvG7Pd|CG2+p}7_ceIB-tT=8qdjbRVo5810K6nn=Jbr=M z?d~`A%*ybFus+nt|NVcz`IpD*zx|agX<5nhq)g#|lR~7nX6Kr&vVFYc$&)8f{`7}G z;J<&wf1f=29RGd#^pDT}^oM_W`uU&!@W-c5KmYs>Pygk~vnNmg_=kTXPxkTh&{fNd zmjCkP2`z*u@9E%uvZC|3ht3nc?7zQ~>$lb-RMLkjT7_!KN}c-+2BN}h?Gpr35oPap z_fMvOdNzGxD(dy!GfB$yTrV)5FQ$nonNX_D>WA*i`^{uB`Ay@xcur`+*@ued5H@M6 z^*#N~9naI}m_Ot8O*BzOpX}@*_NoV-b z<_C(pn1g)MND=);BxkjG?6ay_S~N+WgfHDSmg=6M*2QIGeCV??h&kYDqijde$skQ( z5iqEhUUB!PHTu_%%ZddZM@&_UR~cDY?wa3%X+23pxONyUz(wBP}Ot zVB@C1<(l1_H+>qKgAL3VdVEcsvzS{6|4O7mdToYGbzxRt=dJo0#b#`U2)JmS$;){+FCLHrS(f&r;=4lJn+z~R2>4{Aj zQ*!$smdc;c;q#wwPkVN($BcT%Sn;wHMZwY=}xm{3A!tM=5C1J^&sstCHAQY5gr6T~`msT7GFk(X?NZ^BT zRAg_jOrB_iAdQIKow(RuLf#nUIN#xbkIW-{6(W6ur|B!L0)-AM(?z2)Nm9_VvWI$e zu5KHqa<$QJ3 z3(<(TLUrTWZ4c**8@W#NqcL!=wZu+M*f});#yTF+C;c@SB5b-DzwTJV3D8a)@lcCR zWs68;J6IaqR!MBpiB*pUEOr~e*tE36#M(zk+s7xQ#iXN!lF<%Y*TWZMLpFG}i~ZHj z>|Ymb3R^W?Igv|}P{}4dmkeTO?oqycfAodouJ4Zw2px40Ca|Wem+t%A#LGWdv=}W} z`Q8hmxR0_2_C$z0_nH%=d*(s!;pA%u_ER|X+9&n?&b@+4`5;Q~?r5{iTl2`RQ!Zum z2y7C>0*4>7ZRk7ee0=TkuzNi0b{oIP!*1|h2RiI($;!~G?eH7($lmykA^6B|^!3PZ z9{EjQkAC(1CM-CDfO;eDSnkh)?+TFJ=e*2mSb%J2(IZt-DYC$v8bQ){b8jqz7$llA zmKy64cWN9o7C`L>+;BJzlB9V`Z1#>WdgWN3zc!&tDE=2EOK8Q?=W2Hi((D?acrGhO z)19Q*0Ovgj*B#AqqqxPmGxAg7vU}&SQR6kbZuXwI>fY*Phl^JYbI}M6I-YwnK)nB( z4V6L!i#mZz zCYGVjC9BRfy1(i%R8*FF7E^WGAkC3{La~QT&{UqQkx~DfWPHw&b&|33->N^0C6#Q5 zk4H?I$TIQpj-|X*tS55yb)IShs#N_mRrN~7I z(KB&9b5`T6^Xx5wSV9erWFk;JunyY){Gjm`)+7;G##7d0P2<{v#6@+hKV2`_#T&k2 zvZAXZb^<$3-I}23W(EU}HG&`-&Y)D58upugD1>b4cJ9nvAOiM&_|$3hl}mj$p~fxl z$WXX4-MJj@qzw#nY>N$QuS2w!n&Ki8Gn(m_4AM2-8J59lRk1REPX6oU$?mx<37j{rup3PMyz7=kv<>EZy(r>0eI&)jJv9d~`Kv?`*(P?6TWR6dtu4kb#BE42V{>G&3nsl*h5+bSdpYSQ>zXmT26Vd4>53<$WDg~WID=sQ zY7AtPta2h_h6hVRB)0UcZj4E7^j5z}OC+d!!1y z-sj}@o1$Q4LM6kn%iGuD!TznYZ?#kz!#VNNF~NVFKADo+m!&9P{aDjXzu+sz?BDGd z1gLJt>OEC2XgiDE;fVUZMR7H|VkT!o-bLMY$5=P%x_ir4;ail57r5nR_?9J=%a!mO zZzkk#R4(5H?}DAPnNZ%n4Wx}X!P|gZL??kw|m{{Z8Yp(#VZDJ1g|je2wrd!j2Ogs4ASBZ zQkv==T~MA&BW6NA(TA#}o`6Mj*&>{968TF!L@x}@k5?=o1MzK61gLwC?P~XnMzL_x zOnHMsLAbG&ZHyguJXrkjeXa(ceIN-fs=8#DSBb%)HZ4D}c4Dgrvx0(wbre3jex@lp z4>o}rs~#AG=+uN(EFT2ZYOZV8fUmtu;kf=d1V;kA>Kr?*%xY7uE;R~j5&OVp6-M9I zDz2i*OH+XwqGFtJA+%>kBD-$eq|E9IkZjhbfAoWp!DJ(H*2MZ_ zmLpoR9bpZ*>Oful_JuogMb}XO(s(w=#r2EJOOi3AH_9`e6rr?>u&eHYWf`FoacN{C z%J80S^waO@jXMHP#KJlDSu#?Y%b1OAYrf znXEHV=kzw_5_GJ}V@Jh{=Y+ca*JKd@htE^BQW3TAR`b%^5E{Nc7|&dlBnQR5$5yId}An9r1Mb6ZFS8IUtg%Nl_IVt+C-EMw%QwMHk>A+vqT2T zx?yVL!qJ+by4@nNA=mkGOhlS_#2+Cd{Cu{zERi!W8aMY?lFOHS2@{E|ophM$Ns;xF z#uP`1ERV8#6{S&SHeySfe9A6Wa~+_pNo--0kz*cT+~hD-mA0#^G^WZ)Y@L&sN+%;~ zooruWDXzxS;i@csJXW%wT1#73TY6+q|6J^8o9avB>o5Jc=1ozrKmjaNtY$zs+$Bnn zY)0yB$b`rs9>a&dw?2js?>2sq;lqRPI#9ZT$?d+PCH33Z8rY_*$bB9;?HfUMZi_aI z2oS1NsNYz{O821Q!USF_jnP5j8XN|HG@B_(k3K<-l69`>_F`kX{q)R0 zAmlw|gMxt6GR%i7VeGXPy<<@PDq$(sr@Ut+xkcUjN|e>GSvB0R zQhZo@I8S#?!Rcw)? zl07_HF#(&NaVSSd2Oiy)QE`D7|L2)x6Bp=4*;;f*{vs1ap}~)y5|%z@qYNB zS`|Mu@q0h~L(G1Fw9UcyK$Cyz;K0X_%g2z*$B@g%kjtT49z!mB={<&AZrt}6a`_l? z`51Dkug8$f$B@g;^%!#b7;?FJJ%(I9hFp4X8J=&bP!m_26s?l@fE&AFY);VeTcsoW zM-Fb0w=f)dz!!i4ZMb2bXG}`67PU48Fk>6RcmME1z!(DNl*X15k=U!1<3X}`hGX9b zC(fteO~7{G{c(t|gbUk(o#TeDn5ZM|uKH#|=M^hAv#1l1ORjjAlkM*_%gq>>12)6n za;PiaK)n_?Y#yg- z3o}NzpTzK^KGS~bmb$Q$hIW4a6%;lZoHSj-W5$+T*F?EyUiIV-)Ya zOC8BmdyL{;eSBcFDe4oHNs<(TT;I|B2e8XU3wuv(w%IFjoabHkWLWa+j9HHcVXiCxKw4g zqezKuA3gb$z{y=z&ih*7j4&c&6EFH+->&N(5ois-B$5tDyKO(TU|r%hfquSk@{A@_ zRn^0L;X0E6Wo691mXW9E9~gSsllfc|K{`iU)y?}^okVk-$%uB}pZ~41-FN+b`fq{v z{Q6@ra-QB)AvU4T%!jwF=bQPRWl4J1TR{oTJfLGWkfBNh`>;!Bx#v8x6lZAXA38NKF7jZpDv;pd+{iD{lJA zX9P=S|13y)+-wKZ0ks|2qFHbwozhQsIv{F7^dFssD6YBs;dSF6kRX<-!aZ=mEo8)& zp=Q&pGST$9*})d#*&d@mquZA`VYajT4SA!^rf1Vf-IRIz;YZW6>DjIRLp3K@MC*4B z@aMN(qF7(()daPmc0SWJ|Ck41?t!nC^WGm}YzaacmC8y&ax2N(=nq}-*W2N=<-;|_ zLtncd=FlPf_|+o_DZVH}f_p@m#=P3Em}pp=B({qTWO^46U%Ovr12c^%kvjc z`_cuxk;#pzo_nTard@6C5(k^vNqfmoOog2j2+GC`ahKK#d`S>UHiTpHiP-vOn_|-Q z%)a}=X@!K=l zeFCw`M0dMoUWRgat25D%E;W~Sd!StFb!ovnLm%U80+FSDYliuD1HRc-{f+w0)j#Fm z^2RjG6wu9*x(QP00@c%+D5C^cWD)QvtcX|;lxsTYTh+TPv2vdtdXsY7cUE}4^8<$9 z%fA^?Z6vs3+q}qp+R*bA&Nb%$A5ESCiT(%Wm2)&Tb`iTOD>7Ef|4xNU=Qxd=HR4pLwX?UhcDTIVLe-yvgA{c zHFE%eJD6@U*dEvcwi}^V_xy2)jmT!XL}{3{XX_efhgw-vBD-WhV>WtlwX*vXPfE`F zQE*4MAxt2``lH^rz|xl1BA%t<0xyXD&7a0`-;pe_Q1UbpDg56g7kozKbc0vf-o9~m zc6RpV7hjnF{?+{V?9Fm*$kIp{(6FS@5 z3_PQd3X4CTo#C8wvBtg6NrmTg51cbI%Kv^KuK4Uw%40J0)+x($)3MBoXQ9$~3p8Y%xu^AYAIN{941jk0z7JN0r;;F~T_| zZxms`gr;|Y1Aor?(MNZbrH|2*LMpy`MNC5MQ%y>NV}Il-`DlUzweyXXZRn5fVwn*! zSr9hWM1EG#EF}VfyHBsrrjI_eO1WFbv{|mDeAcZ0BkR*bc14}b{YVo&JrT&{dyKjt zGLYPaweZJ9DSk{)#|Fb+%(h(V&%@7Jz5TBG-7{+4I(b$IoVke_0=wiwy{JB_KQc3a zp$sx~?c*QWT7r+!3)jC~b0@nO@b<*+!`j$BAlI%;1K~Mg=U10Me0JUVb&E~Hd`s?F zZIUgfWK|)l4>l<|p_b6uLdz_5?X+7Ttzp?m{mfcV)I8O&M%u-a)UM6AFO2MNbUZ+~ z7L(<3P`iQ4O(FS+2=@Yyn*%K_ugCBt2X*tmw z8+QbCYT15xOEz}otLMCof7u4+YK%{b)S@4{G#ArKjrxyOrLC%ac58bZ_T{F0-*07V zv_c6;XiFLzFam!L-~EH9HL0R`O96zjf|T`$5-iDtGN}b(Mc1V7GW^WY&j(*)Nt2%s z^n02e;hL+naxB(r&wWj>Afm7IT!e?IbNk9BR3W>_*X}`6#u3a^CI991&9RkO0dmJ& zTf{E1>Y|aODOGO+~y{U9lOBqxD=@sp9=Nr;zF_=!Y6f#@a< zd-DmqDq?}_erX>=xGYs`&DmJkyBJ-B6P^|jGow|$s)|*0Ne`Q4)5=+vmCLs8?(b2u z%!6nD;gu%@m!X6~PR$9^i8}j<^$IZ$iVDopHpw z2dKl;W5Oi5Hmlkwo2Sqj`OvXI>oWr9!B(!tN}xdR%1 zO-%i+dy&E0wY>m5uWv|~WyEgLjvLYi@xiv}2b>lA-Xmj2D>tc2rXe57CK>Ye%GW=q zu|d!4&zm9+vepNOmsu(^GB@JMg2--fL0lTau-+`@h&tK(4qaJVy;nNH{(k*fLVCBq z2TNr4l?Um&vj2`NJN8Ab$JE(Yjr-Ztug!-ifJEa7B zq=dVpg!`g|JEDYpp@h4jg!`X_JD-Gmo`k!dg!`O?JDh}jn*>cmfqo`Cb~5phdsOBT zMD{R==u#5XpCnwo*xQrjA-a*o_92OAvlr|=vSZf~dknox7}#%QtKfYv;6a7(!-da7 z-9>ipD-zW?Bs{-}ZXvt(32~i3dN+G`hmdNIdV$9G28rnYF{Eo?e8&%i?H>E}fLL#Ur{@M1SH=M9+=~>DIABpN?85j$W|Q9XdLDaQKAbd-UcA9Mg`~Ab?wkOk;wrh+iFu(^s$&nfwrXP8%`o5(4<)*My0ectqVnoaHCMxB zb>d2J1q*Q9r-Eu^re(NP*8Avoa^nHYWJT^556cN2i zRp2NkEOp74CJIk$fxbAY-fe^o+L+|9!&~!cvl^bIXXa0)6q~;+v6-&TQtX+s*Mt^s z9_Y0x!S}xxF>zssx8~tj&$iC$`&ZBO{?CcBMkaW&&*rZwYxhdM6%B){^#znUg}e7p zh8T4G+3iq|;i40hOt@DloV1aV=_G9PYM1#Z5KL-A%=A%|x5iX0GP^QGHXI5iWg4sv zVx#j8m>HQX)KiKnlwi}9RD|j-^3y)cRXchFtEJwHglE8&U2K@)&en4bZmnAb@^+`F z(OmF1o48x>(()a21@F9iw$lfFEqNQC3-;r!n(?}(yFJ%#|E+kt8}Z(!3vZ(bZ|5`p z0RW!%yT0bTfrBcdkm7l11lQNkN@Vsz?w5`g!w6A)f!*us$&#aW)z*zEfaI^MyY<3Y zR(C9`JC@ZQ%j&*oWp({6*PGPEwleQJ{iFE{f9TpRTdCQTr| zB;QT-oELM}=WvSwUtZQOm`DVPm$r+jUUQ}&mjS~{y3&a&Es1LkwBP?+@) zaI-^EBy7Y2TfkcoO#u>D3u+bPY6-Waf!L&4MYO0@KeGkJtE5W!SKH$21sGAGcR-t< zaFyODuRwL1Z1O;?_1KegP?hvHq{*|6$}3nVC@2Ip$r5B!DTNEVpjvIVbi{|vX$qG^ zUws#?16rn4OVuj>wF*7?)dKw0zSiF^#wx5JAt#tY88F)fD1mx~^fyu|ZmEeRsn6X5 z+4ACl@!8>_P5FtO{=fL_S12?Pp7%S*is(1V2XTML925T)Y2aaOu89|9XCBJ$m~-Mr zY}+}14oYP}SQ{31wMLE`)M)i;`#I#^W9Y-ifrUo;R z2~&d^{bVS2rg}1zLsLB&%B88E4CT~RPlj@9swYD^Hf}_VY=N8`H+Iau3DPBNa37L` z6NDMg(QzXtSI3QT&W;=2+#NR_B8SI~*j%3G(jt&0BDbe=eepRy!L?PXSVMAsVmSIe z0_@$0-pF7MkQ+Pa0tHtblPffQT@krMol6^(Llj3<(F>Ii?Y$i8$$!ZEN$!0z6lHC-~R!`!y zoWejmvY_5AF^-B-4WEC1%R)Cg3UO~WE!?veHLiDI=LCoJFbo_-qr>6gUWRT&QNA5} z8-__my%>5-kHa9uRu9C8UWabPnpIF-M}M?i55x$l$n{AK^g?uF06mWDjo7)o*dB>d z3k1Kwpk9elU~S{SAw3hz84cOQa1TW{2KQ2oT4zVkLvK&TD4^SY7rng|qhR*+U+nC$ z7zHu952NX|7zH!fkI{`tDH09Er+7grCcDH8e~yWJCeReHIlTb0N73sD_H}e)NYHt6 z3K2WMW5371=Uq3*!F?Y?%c^&UjP3szfZ5RRCuVsv;tO!umtY(1{+XE}$Y9I`J@$G4oOD7uz}AD+^GQp_C+w zifo$;WTTEL&ayK0%XueV44CiRq14(BDHcHX1_O*v?9qumILdplRnHRVMo^q0N;)yKb5(T^$3E^Qu;9wYKhqOK;)`YwQ#K{@k>wwVlY-#$bq6Q1AUNs5e&D94l*%l{H5~Fjm&wwX!A=F%>nR zK#`WZRAB@|h_y0;Ny@>~3lC0O{i8<#WE4O;#|KY$Mge3LKt=&%Bm|=XvZDY(nIuUe z$n_n~e*oJ?ggEk)O7OwiiXL^b1n!nc>Im)S9oa_v6n&HMY+TG_^-#9xvJNxpBFU)`c zYW{on>F4m@PamC~efq_tKRx>V%P;===+Wn&fAQ!~XP-WL^yR02LT7tgqjwZiVezN4 zGn{iS*0}dMsqlR6fpca?`QHy@d|XmsfnQHeyR||RVSU=I&TA7F!r%G3_1W~xPp4E(n+UE5CMP-ba(kPksZ@@0rk3P~g z`!RX~m9<_G>m7gi>-m%ZQjP2%BmeH7jVA5&^*7 zr`KoGN1xd`qHaCYX1r_ptl5c=EVD#gYwO&Ck2HzUQv!G7W7PeStrKXcjulyOQOX+= z)UoOC7qdZ?SN?qA@ssijg&zJ>EZD&_Tkp2+ynjl z9RImE_O1(ro;h2lbeeoJqo%62fXZUF(pr=4b_1F2=t3Ow1 zt2gQ0asTuZCzz!ZLkgRF*l&EPq=<9Tm8RB8=4m|bGO;94fYB8!gvFvT*{1W(^k%(} zhn3WG=uAvR)Z*ndnP91H-9)B~Da5I*aZe!XlBOwPXrAGPbivJnm>TZ&W(($QQ?RC` z&5)=LsK!nU*&vUmDyg^hXC4TaSP!F4#Wy+&MV`pra;jN^0y40esPgFbU+ zu)$Y^9S?X~)Dp_AwEP|>|Jj)Yu2YtvfNQR|Wx`q3Z^f_fAD}CmOsqj@s$@EwGCs+8 zayO?LIo@q$J|o`*CfG&Lc56ppkxbM4q&Ho-CE<0yRubf@L4DN^v}4lS=KUH8uh9_K zu%}KcwIl++?tt5Tu;!ib?wASoLKwjN#(rinjcI zbzhHu$VlB+2T|h-^dib;LwT;t1@jvi<<;Pe7I&`==BsK^%rr$jbw5mU?eak{3~WG%>=lGpw;UO_UVxf{~Z1#dQ+iBZ2Nvx=}PQ-GbNUh>x#X@JkI5ERS^rbvO zvqEW(1Dep1YgU5ILBJQ+l$)Z1oDqNJvy~<_mH^eJWYo3_AeE+TUb*KUkT%;>e_9hF z_+cRkux=!amHymoLKjOVPY|7wsnJz&vJ7menaYX~1yWl{=xZ2Pui13uj6@!=)|_)v zQh_!ubsm^6`;4e$dVGR3`z(|`6}{rwMk`PPTO*d>#lmJuXLhoI+BELqO$o{rl)wYC zi3d(7)E~0B#jsm8o7f9ib9G^a9Dvy_(e=Mda7XgEJWdbDC0#7tIlHc{P0YDR&RA1H zVpatzA=fcM5wVKc0b~%#_5g$R(N&(&ger@p=nNeJl2Uaj;gU^we(Vp4s-w zTGDdghL3omx^`kQbBHZwQ}xCiSa20EtgN@H7j=TLlsuteA&S5hGn(jUwtSy;QHK@A z?Odu$eQYkdPHG}xf+D6N97>n(9r4zH@{Qi}Y1hMzf{1G88?K5Q39?Z;_a3!z%@~3S zW<=J>HDfyq_MT;O%@|fJ*NlPHInltvxn{JgeB8fkuAO^a?c4znt17#;yz5D`d@lM{ zokS&cSL0b!(OtWs;fE1lfY*#RtLrJgi!S~3+gkIEE5SPil^a!Y`wQ_709h~D8z{#+ z0BW;vZ&#h*Z&hrKU^AZe)(2kVH3)uCRdoPFD-51ke}S{fgJC)sM1jvFp41}iHC_-g z4OJl?0J2rDIKCFKol!hXGh>I#Hl#YS8L>5r&4{T~?4BHS1EQ-I*B_5tB?v;_fJNth zkb$giX_SFR8EBM&Mj5Dcj55$D1C25e8fBpH<1woo0C<VfIJ zTq)wr`W7@sO(SRRfo#`fXPrO@g5sLMyP5Nqo+)L@k!3w?;&#^q+5|SG1+jfcOhoRn zS-adwMOLN`q=5hEO;BRv_u-K>VXDAcwn1b~ZRa*UAqX6%%J@F-TXmp)TZc4ndu5Sx z+N+}}4KZ}twOkJm0w%qD^$;!Ez$L$ue@ol_l9qj1WaAVZz{bSVWJ!91AN1eVk?@^~8b^1}K z?->8?I{oO^7-gVxj53gCj51KuS1mOikZo4aj8)F_H3c>gQlI%wzGyu*wd zjjk4{Ap5;S57#)(xT;Bd)m)R8a~O@-cS?wvNO&&5lj7n2H zMPC#U%pr-26kYeYZ1dVNE~Oo8lvffl`Cw&8US*Lr3>w3?LtE>UZ{N}dR11? zXd1r;fkX{c*-i#9w5uw(iy;iPx1l_Cv1Jt-xDj<5TEAs%Akc;#ZQD9F5HIy;nA6_H zAq_QwYTJ!^wdUd6afdVA zmfXyQjnuKixu!b*M++Blf&M3+yJIS()*_7ExDLY^dupGH2CHX8CDQn`#u;JC#y(n& zSkJaCf5uab*Gn#mZCIkUW-8Ia9oZb5wCPeb#DhzAU|fn@G^mu}piTDy{H?bT-}dMd zfrp8<$m6H-AT}Z9a^c4*Fo;|>s381WB4#O5bMrR_zv26 zE3Qve(v%2K*PZsT1-#7Z6}FmL#`#^5HyZ0}?12Jhu{XP1rqsx5(RyO)3)}wv`jfzO z;)E&CJ#U}YmF-)ECAtNfnxP{AetcVPSDcSvX}gE(!y%fxTMLrx*zGuoJttB&F>xEI zK}QO_dOMFWQ}npBh)faS7AQ_BXNStn9?>|5R(m4uR7y9Qu|^wRHzjjiWN{m6&REk( zC)`-Gy48K=T+B=fT*b|lV~ra#u9oeqrAb=@C#L8;lRS>*wqPY?ZE1a1p_yg_blg0p zJ_>2d@X}I%&hjO$*+MPlj5~b7@C=N)&G}xknklEObhyAOP}IiGGhS#L$kvC6DGfTE zQK<$JYAWwbtJ@Gpa{b`snsL>n*ij(uuEtaBD5#!887cN&DE8I`yC2cs0yom_?(sg; zZRR=#?bC5$tfD5m)WMHCsxtl%(xPex)iO^D5CVQ5NB^vMvtYGW4b`M9EsM%GtVV^0 z)>Ccjx>t3eYaf7TiB;BQmbMeeDyUiPK(5i6R+Yjbsl+WgC}F~@1hO|3ehF-L){Yr~D$<5FL)>D_UM`bjG!+NNaK=B+bQWpSkF$Qe_8fy&je9SIu%n4*&s zN);xY^E3|08dbs?!M|k|NtqZ?VMiR{eZ3^2v0f@&pB;oH5(R_)XPnu*iAbQDk{c_p zy;nxxgSCUwSDjb<7C0gRZ5nNPAz`|G)~_MF+Q!)$Gg|tJ&GntY%j^*aBtkJP@mb zcd=#d(2w11Xx&i29>5b@WQyilm5ODiml+Ge+~)Dc-h4enY&@s+>tmbyDk3CMaGUpr zv&@vP^FVY)1(QVB*&EG`VHs0T%Let8hhCy1&1kcY^<6va=OsPc+}oJ%*2GDorb%(h z!jI&!U+AzRah&+FEt+^?cZn@0k>Y|Ea4AYehzWnM@x1@^LX8j*V3SG_LHSb&w*&*v zv$3Z1L3vvbk+bzM`C7Z=YQ57ki2HJh4RD9W+X8kk5!CPA8fygY=rinfP({@Ot;W5a zQ}4c1kjEkQ9@h*4Oxn%0^sWm4?a7_Ae^*k38_9lc2N!9e6wu&>_q&fuhtL3+9@o)W z87Jn+jJ;EkX3Y~m*q*j++qP}nw%yalv~AnAZQGo-ZEb(^`^Ro<#9r*h8Xsv?4_{!vA9nPgu^sf(4g|EvFis*qZx1ZeQO5oU^o#}t zxJGC@L2|tU97_xb6GBIp<~#ZT{p*Ypgs?sb3jl{U46m^37<1bJNW`NO{sdCrjb~o^cVx^lyogS@zsBAAnqY%#Plk8+$LP z8b*o0>wE6_oO;rwII`D^Jhd&JAJ8XZ`+yrK$xNkW(Z9@>Xw2m0u&<}vZ8?Oz{<{0I zE*OD54&L(9bR0(VI1FdA8_8w<)NuM633}y;#)8_41+^LVBaW!=(t7^b{to{LH!teD zwpKsdFp%4)(M8z(7ppUR2gh&wE$n{|uxEZ8b67_v%ksY-HqjIT?9n+B z`gZ7_x8vVS^Y!056ca-mSoI_PbbtT%bRLJZAID5jgBV_amG=kE!uDT@rpt&8Wd@O^ zkDTkevWJT}xqI3>t9fi)mk{2g;(Ssz=P9&HNi-3fE@W-z`vv0REfx>1{gfyKi8*`P zg-sQ6nNsl=srp#R718U&#toMwE>KoR1IX2Pad=5I2v87W5LAYF339M$fg?&Na`PKoBHcCwFMfn=s2P$z*-V(94$k@%IfrF}Avo%(0ao~@5AJ%B>jtY{Bxn>nL%>=S|aZ&2d#g?m+{c>Ocb zJb($FaGshaMThgd1oF-zy+TX_@uY7`Eo3Ll7ZXw2$s0&A8s|N)A*Khep0lq}wU?v_ z#3a%R7f9P!So)*YEb{!$?Cl{5Ym9{ZGv~S16yU`I$a-cGbS~$8F zf1x#k4^?pM+jc7`1ZN`juTFke?g1);lgNd5qb3rs*fjy1?EE5FkfHGv@@$ zLFn{_PLni#xWBOcDCt2chct~CPvud5{q=JbP*T&@T42pE$8BRsUdSCB$oGlo9)KN&H}+LvZ|_Xdohxx^vx1O6c^%{^7yQ1klZz2Bz5`|Dz>iC{r9 z(QoXGrjfj+2#y3$Lsel_UnC+lX_9>=bmR!@sL01i5czvk1f_h-SSEvTEZhgV6~2!m zmDa%bQHD#PqTfWXYmPds+B-gl!cZe)(B|7aq8G!R*cRd!Z3l$nA?5>AFZ$4g#~<$? z1S!TmX++Ypd@G>JgX6#&7P*(#T;!=(>bp!Rnj1iQJe}9Hypmy_Pjw-nr`vl0ww2KJ zG=R6n52}zNpG(|jY6Dp`5h@UUJ?W;t^A2vKpe8D9!m%qf2xT>wPM?ULz=pAUrND=fSR8GS0ufQO?GNL_|2R z@~7hBAr*z$=oD8oe1>@vBm18G$qYmdLO19_i5HQ?tLXu6!nU>F=o}cw9DRa>zO@gs1=N8jQD{0Wb`B%+i;Jt?`Q4oz}q-y3n!0jfVp1N{6%X#g#XFcVd%O2Mr z)R%C|8Qu3#)xer#vCDwQ%?A-@0GRD2U}D~M>FVs=mH+mbj=II3Vl7s#%QF=}w+=rX;ZSN~SOmxqT@hAiYE6rk zW2xr%_`C8K1g5~6cjOh|>u#`1DIrNd9L3hgZd5N|D4>)dMECcBj`E;R$*MA>qygR> z0G)~Y?ppe+7wJPT4oD7otic>O%*H7@0>n_L1d1h=1cf1oY;t%~vV7xUP`*$}KY`L* z9$p8S54wv%XyUqROS|~!yAbhJjaJ^)0~K?8makWRLs6DUt5Wr;4G^5{)42&Q=(s}e zLlhsn3EqDaHt2ZVMI_MSyPT6Dh$P`+G@*%Nm!yhf97ku?W@;Pgu%a}V5KC{{k-pX<;6b3FOgBHrDUS?Ij;gH5^-pd@w z|7So^tar)BeUs$Nu(2OQhMdlL?3qF6t%nxvPfL^uDrfyJS49{4^%rUauP`G5JDZ72 zsIro?3^IG3-D8OKJHlb}J<*B${xE?j425@_pARajqy|Oirp%)Z;{4%<0x3%0n?QGQ z)@S%S1lF)#jjt7{Iv1MIwe7p7)_a@1q&nRDV*=+4Nye?SlZ-FX%t^TGC~=h4qsym; zlt0OMSnBHi9&{Qnc07PS@u^*w!780bbKON|J!f(yOiTR9!s)S|^o(>qrN`D`;!K^Q zaP7#+Nze2)?x!>BYmDC4$HGI(e$8%%9=4zsC-ep*tI7SkGLy?3uixzCpGP8Nl)rY^q# zZ__s%djo+IVUiM*C~g{q@dei3Eln=*>wwLK^;*6cGLlb@{a@1KcS;1E9PWy|r;PX% z3nefDGA0_Kn7vo@87Oi#t>PL3`U0t6XcAU*8}r2J@-yX-Q8#af49e}Pt3qtx5--WJwD0RQul(XR#mC^{6Re;zanj^es?eN zVK(M~$2(*q&JC^pn3fQkXvyM>iTnPl&YT>kgA44@Tj`Z(mdY!~S_iu48T08SK+Fj2 zd0d{6RnpV!@mpt1bue&LV4W+(cS<>~-T(;)X)j?yYy75=E+|XGe}OdsP1}UeDoG;J zB`2Rta~{Hb(%n-HW7H}HvE){FuC@m`DmZ?|%t$B7KT*YeaK;azJI)H*C6K|45UGjY z8c#rt=uZT!dyb+fVa12Utk)d4LD7vBl7s0Yo5?v-`r`$+(uZ7Vtj*C!nw5`No2x%X zai|@}%kj4Nu~b7houl`l#xeUG*g3}V6EKMP89&7dfbS>{Xh1ocNO77yo?7wjQ$=A1 zjpNozcqpuPBP%S|NJkrwxAd)3lzB_dQ#nQEISkglB+KdjVVFk@#*6%+%3)u&B6p_K zuF^wD+uf|{YPJ!#P3^bC6&hf0zLB_;`}MVU6mU+F7GbWMq96h?Ju84Sj;wQHm(b{H z-^IT-%~($gG_Ja4#3c|wT`SH+(>>r)llZVJ5LaJ`x^Jl&(adsrG_MXE)_*6PjY-Qc zQP-cauEWEEcuc%kfns_Jva!KHF?aD`q_t-dFi}FDu^XlxPkqkh1yUYANGjqvPPy7P zsPh(a7mXFF^bkO`u|9*!0d@v#b+)qKF$Uc3?r?vWqee%kw+T`vKz9O=6q*P2nYt)N z(!ZFCW{&qyb)SjKUCPgK6K;r%G9UM zeNMnL(3Bvc1Uk>_HxUzr9}Gk`4mAk?gXTviLvk&fStz{a zniN2(5pVN2@a#|^upA>Tru%I-!f%$rXHaitDK=>|R=jchEyzXURn^~AT+}B!8Jvx} z=(h|qMF*?CFE7+j(RFXW> zK`5T7YaR zw}nYzHnz($OJcGf8Nq7Qgnu%AOFxRRTYQ1jW+6Fiea$ycznb6Eop_N_f)Rn`F2DN?xYT6V8*qr@AZ> z1C^I?TqG5sQTeBC>+Sf)Pr}!?ky$+j^NH>!EaK4^RyKLN2&rgU$*#oW>Bb~0fRM^s zd!ZZ^eehB3UpoP}QV&=^?4wNf|A<589Tr+EEd_3H*-*+2;$^AeO%2daVTiI)JK!OM z>}$nTTIxyAiYgdeRmnSZ_O*=6q5?U~m{@C}O|sGv(E>8EL>mw$Qq+T=MNGMYJye{) zub0xujY1VAy>XyXk#qC-amTZ9q~yM>bFKV^iFvDlTo%KazS9%q`#!upc&R8L!WzB1 zUj%8v_I3GK{=7-q-!CZ0>Gt}%oLU;9dw98&t5n}g%lY|x5~lSD19ANQvE(W*y<#pe zI=&bb5w`X=0&julvNZ*E>8Pj6u`Q1<5}&~XP!4AxVk>wJN5;74{XV~HQjqSTUBJqm z%QeuUeE*za9_%XDELd=hZ~D;}^(!j(Wwa?OIq=X#i~%kp`BGx?6;5 z?`i&0_1Om`g|NbYL4{^=;FCw4T3PcSZv80o6we%0(Wa`aqoBHEw)_PUt@Q?C+2)s=- zSs06;mOy<|eaELEVDk@#&QAznh$;O6QzX5_1;Q*5O(Vq7-6mX@5ufdN&8jnw>;zN;$svID zn69;*QBjZ2qx#nhIzZ_3+z&^%I<_Awb~@`FT+?L<@VOt9^FDoC_V#Ie^pSyoUU}}L z^B%$2SvFo~UN0?wOam!Fs<50J^G^z`!76E{=Od)%M4fVHNdBR9BrM6VQZNQ=nqx&B zE0GB?!;DqeLtTMbw`3Dy5$7htGS$j@n0_Y_kbVW1hhBsXa6|cCS8P4QWK-c>j^Z?x z9nA1NLPHUZt7+!SkXtT))UwA};ys|dBIG}*(C;N@!vGbbxs;`r9vU)M$+rix`2iH^ zYslU{e-OLJnR@m3?Y>1X0^Sb!$VjrKA5soaN0xGS(RjPDsorAQh#~6!7Wcn~5I3n} z$41z2LQuj!6Buja7>d9a8a>F~g?YfyLrpJ!q$+@^%@vyqYiwB?je%LBgES*zNtXG! zR%0>8Tw3U6AUSg#YUdRUEDY?^0Lvvp;H8a&@uzFIYqSBzS392Yi^dN;fGD7BTo}&0bl|ARv2Qfju&FyUrAPFlwM(R4bmg;kroD3uTQ1LbIff^Do z3zo;Q&X^0TGy`&mxaUbr;;OR3{nF{o8y6iE7C3Zr21_+I$LvQmah8!y0Wq_*$lQx; z7_$?;S?!`p_2wEzKv;{!fN2RCahr%i4Q6S2XH^EOb?NX^`$WL4VVK}0BDwcKHNXbUPnqrzhE z>Q&KZuUx}B?Vh&DSXocKkVyQNk~uRC)B-v5p$q1IIA7pPV$sBx!K#5XYYx{=dnM7l zbT>wZem|0wy`U!!8HK})wGucLia8EMO`P9wLpQ6^#KT-zKC`4+`PU)>y;bM{J#%Ju zeyefL+l`%*cFR+i8P+wiutACD{oG3$+#NOSo(6n55=!ivM)EAGc>Q7iuIF0^n#~O3 z9l7I`+VW;7>>?&^F$=qR1<|wY>Nda1Y_t55=j9%uq=jQ;Dcyu_&lGli;fV&g2i@7G>UX!PaF_oM8AsZ{0>;%`8 zG*#5H*YL4=J|iH=TFNE`lOWLH`9?$7O4KBdUJo2P9OK04`D`#KB2Tju?>^_HFnGYe%~`L8j!L zRrV%~vP*fklallO!8rdc+3nv$85A&_O`=MZX|rkG2RJL8h4d4gF_pAV_JOf2D{94B zO#=}#=7PLLA!A8Ja3F^D$dzzltbX0~$0&)RX#&BOziz6$8Aj>;9*z3N*6<(^#2yx; zl~@J7Zr;~7MNS%z?|~G;7`B&7@%PI&P9Ga3E@IJGcHM0G@+A z0Uc}XQg0}y?)DR^9J9I`EVqE;pg@~u#zWgl2kmAu)*6F*eDu$=lek=V?af$pt z1D8$;ETj2SnHk5b01`|{QI>cg_ z61QZ+N^ytDnhhr=0C}_G_+$ckXt3ZIsBeBJ+L2g#h@9btxv~_`)ClFy4)J+EJCleC zF+36{H?hj@gVn!}H64D(FJ&;)U8pVsEiDc<1#DntiOaG_7QUn1E2BjR1sKFd#4H3~ z7lB@n1>P`hRSvS6FScq$$b&q_sP?Sw`^i8XI-Woxaiy#Ay%a}Wh>U=$OcSxjyjP(V zWqkB)ijk95U8^i?Ue-5XDUqKi-<(=KzfTYRnwD0%TOOxEbBld}#oD*CkX6rNUSOD@ zt1EWicNAI}z;I_J?;1rIi@-bf=T>ZZ`w+d$xfYI5GCJ&;Z~4=Qflw{-nju{e_J#xY znXTXBk4f0p@2I%=0ihRkw`V%odzvft3@16>p+7fdS6eCUv)GKie_H9j_X1VGqVF(< z-=KcIKpK4pG4S?(zWpj_;S?Q=sQ(MounSYa4f6z?&o54H$tG~Gc?kxded2@K;iYl@ z_rpiX+h;fjP5VjScgKsYM*z(ylZOOjTl-IzZk3{=))h@b3Tr`N%IXYCR@ZF%W9Hl( zqq2Qcs>ZPElW?3$(X%0=u$Nl2%;^2>z{@Y+s26R7kZ5wJV+U=dVycKhJP8`fb2hvz z4*s5+HUMaB#j6oWBmb=#Xe0m0h)RoahNCohuE~OO<3plUxv7Q z#DZoD(WL0LSdrtSFg=IWlsdE80Q9{^YL_8;F5H9K4>nXgLhJ-z{@bH12%x6L3_DQS z(Ir{!QJpBt)e=KdlKh|Dv5xRfHIYI-7Mz)`-rDG4M)WRyY|KyW11*1&$TEm<$y6b z>(}E@gHK{d?^wMqUNQkRGpP9xVhu!pss2L`)Hxg%rGe+B+#HK*U_2Ug9?s$m=yq)# zm^Namb`OcctxsCN_1^I4J^qBi&Hh3P+6sJP8q}xFIdS*y-!7{-{$<4 z-Dc^WaH~0Zk2EWGc@tE%rSe4ZM0cVHdBIp6c3u`Md3=q>z|0i5DfhjAto=ZQG4ufQ z*tMuN-+e`%26SQ#Pu`DJNyWr~p-~+r!JuW7gDK>P@O%1(Q_0M`;ahx;!JpYt2_2cw z*Pc=uaF`;BBL!APY>wHoFkUN2QdISY--v#z1D`grP8NY6kqbXxv#w(qYr{1CAVAZs zw1uSh(s68LXxg_yzcjPdm*3dUORP9wsIArAY~^Qg7oX*7WSQwt{@Q;MUr3cj^#5X$ zs|LcEFHpr>$gk?mxc7Sn9^r=@W-UYO|pTg*lQ$MH5`T2HlUg9Z;$;v5rq zOe7Q4xoDP!UIoW_#&K(X-&=NJWgpsgd>@dJ6Aw(f?b#*3w=; zPub=hV4(Gml-j}kH|h%OUt>G+Z^APl({RzdpHJ-`>W}#eeSdzW^`VMkxX5ex57~C4 z!>Nu@q!ivi>pv&|6Qj+zwPoNXe2LV!3&uz(6_1i^#z-zJ$p5%W@erWDwkyeBRjf0g z_qA0Eq-I6R&Ev37sjsb0?GDxA_YHVv`nEw1NhOvrR=;CdW&uy%MU%BY3cL7-Wkb_{MAc)-b`$|NIza1zImWrB5?d%%rk`SY&vu|3IuqU|mK@8S%2!K-+x&*|lr#?SEmZ7$yUhTw*SAWYiF3y2sm%VIBS(Icd9 zF*(y^@{^N(iBAp@fJU#~A8oynA_yhOJd~Mh1U=(g%E zBCFLzft+Dv6iI;(i)4oybeIBvQLCz3&#q$A#qZ}?AN$g;kU3y7!0h873jMj!d{h1j zepbO1-#YiIU|ruw-n-dyiI%&6h_*^_+=KJsn85pN`@2M^=-etVIGk8wED=|6+Z8lv zpHXc4Lk6e}VccRaND~Py5^q>28El9&N|62}W7%`XRu1Dj5XHvLz+suli(3I7_o*@W zwZvnT5MhOUphP?-s&};xJqfI<9aKL zqWqUjjOkD_F8c+QYyl_UIoa{5Y`%w=E8Que+>iSxK2x!xxx+(1$XHGqY>rEO{>RXx__H*(tUjFS#ss;#e z4}Ih{ndNibT-r8%Zc2|CQ;%3i9EE~seA8{)>YoUVISIl%PZ2dbwUPH)3a0DDYF%RN zfnaz3&HL;VvG@^$11^U{)3s+*A-aq-ux`AYP*{Ggs3)foo!g?nZW5VkmE5hi+NGe{ zhu-FI!W21ktWILThFjGQ?hk!T+SB8v1Q`Lb9*LDo{T*_HKR_Y^(HXwEm85j3D-Eh# zJQe+~`RTMdE8;=B%Wa+ej5S_+~KGI&kxD6ckT4$y14Lmo8Tp zpUekwQt|AN2WJjj3j9jlcEU1{Ue~=X(T|$!k>bvNXqe-I%ZBy@{S|*0|fhb z6&kI455{qt%qfU-%YA)WcjWi@=(*lw3URubI|3|gi)oc>M#Ff~W(x&@Z$4@t0vMql z$KpBKH-o1$bZZm|<=%T#d=BnFlCPuB=WAVXFZ5r5Pt}eik9FS2ghg~aJap%Xox4UnwI}_tLM_;h97ir+Aw7{pwq$(ZytBcQsLaAv)ywvw{ z{V7OW6zImk6lwVtrTCRc87_cY@P&G2;Q`}{32Z3>-|um1oSkcm*!9Osi?2TEqx47I_F`)CA%~J%ACA$#NdVUkm_Q8 zglg72-V~zmo%pOGKHTp=kW8@Fu&eDPJ~HPs9E%afHE-79lN$K(0B%HgD}P;9NO~k< zfO}TF)ogHW^v=2f28UAA`HypGkuVB%s}Pa<{$wdQ zmsNIfJmJ{J&y>I=O3-Ap(?V%@%lW#GtT1W(`=|2L3hPLM2D1S&I?o#39F%uS`-gWN zL?VDuD-6*?&yKD0gk=wI{usdI%}{`FJ};7EsmKRRcZ%l9@uw*0c8or~)`xwA2B<2}C;-@tm zqvK$dV{z{3^ov(r1)dh_Z$ErC(ja@hNPQ(+tXqgchAIkVv*lLDXvZPwTJsZht5%-X z4p^5<%aagIPl_-!Ve-y(n)iFej*Mc&v1i*#!HR6s5Ec5RwCG~ZWRPBc0FkoDeDS|$_hqA{}!;kPMvFX9~TK8!^iI6P8IJoEQMN{$)#>0LJlgtJlc zNpxwnY5h@mNy};_q-QUq_|69>fl%5Sc_*ghCTXl$l$Id@n>{~#acC@l^wv$M%49O} z*hypRbu14v=XN1})Au=i?LSkt3)=d>RptSVgWCxFmGcA5Y_EYy3LgQ`>2$Xjfp-`- zQcD?-xbHg@UV$6(5gj7K?xO1#k6xFaNm}RYuY(aNOHO`~>**08#>~w?CuRI0MG)KW zKY%X=BfymQF)>y)%guP9sTat#jlYDt=K*G3yw5gA>3r3Ox3;0%vEUit+qF53a1BV&9Fa(Yow-K>3qoVwP-^#@h>a=CSGHoiAqw-X82W=e)xq zG}%br1n1hSF7Y9JVnQC?aK5RRjct>(;_c(T{TrtdC*xRf_Zqx}iqeGqLNI2=Adc`!637`~`MFmOax?})7%ob;P_*_S<`H+rT z0*ZHm83%|Z)kUHMYnh$h|(R(GZqkx*EQorXn_9|ZHn0=R-lvI;4 zWw=miG&4r#yV zT7djNO+fjyQoCGC^dNX;zhK3{9+!opPMeLc!%MUfM!Oo&XPul#RTy&1brZN9ke)|-s#~Nnd_U>%@l0aNDq`JK0!UbiiqOmhG&HI; z9LyZ+06$m(RL?incauz65!Yz4<3NM?M|5XNM%XKhz%tP-VksN*BZ?4M5kkVHMmhMH zqTk&BZRycYSWu~^Ztd$quBlAPX+R!+%Y&3;gej|A7y11DGn;IDQQxp(J-cdR zc2SSCIafC!yQ5^3l2tb`{p9r1uA_Hc^#1&uqkek6p+{)BevDNNu4@<_7cayAW*LqA zYZksIS4~EKJp|I=^0=<8;r`Fb^3PoNebJfXXT+_;Yb`Y ze4(woahn5tm|}2Jc52T{^dxOegR9OQ61hv)_|mZO@S0+(AZ4O)qWi_YX-IK0?(d#T zpUI2xA{|C40mPXk;q5!%C6LJy1<9hN=mUuUaRfeg_m*q>hffEx5QVh0RI&4%G8MT> zpoj5^W#1;oPI9__38H|G_%3TX=*;nTl?zaa(Ed@)eO8H3R4Ds<(PX^M*S|~d2Tp_} zy0KI-MO|_pphjLU=zg>s1vsk0EXxBzYq2hBro2U+Pnz;Y;Mnwwr+1n3CYRKk1E!H^ zo>zWq{46p0WxvW9wkAPHoR~}Cvq1sn$cleMvL5}S2=jZ@YU-~V5MsUnN+MSeP$3Z! ze8bEt00mLh0DQl1_3@!~89YUENmzf0DI$;7*1b!^QDk_))$$k9e3ykaS z$z~M}TSB3cvpj7Re>>Dz@VqcR6u>3V+^yPKffAJb(Ny-l>~F9K-IsYRVp=)8V?G&z zLq)tcy>Qwo*Nx+ntF<^XJl{s)h!tU56`UUOm_S%E;}h6m{gdAY>noD(K+2T>1Z}F# zOkL^Gr0?o?(Ordw$!~+MW?JPoDJ zoNVteHYXW)v=cis_!vq#XaS(~a*QKLv$W;<08!PUz1qBap`&CQwLpbT?K^IyFAO=$X5PuI{$mH8Fd6Rk!P^&q!J> zIX*T(OySA}g3F&Z&9jh{tw!+8X9Jk18oYXgvDV7ismxLfK4NLh_nx`JCcIK0deyN?!qY{Ud{V5VGGrw zT9VSKlY2>01(H}Qr>eac!$-6zt55dw+*A?ZGHlx$)YX{0@_3(OVH?%f`NCX+p-OKC-bN zz&j@BSYEt0g$uG~l);-Xc~F@K>~hiM88+ALK`GOC0Ope(bbn6fV*MR|B^nwJ5m}sX zzH{7=x;A>OSb(M?Tq@h@*4yKkZBZ)nWcfws86tm*qTwLovp7FYWi)%;`kAG!GEk)1 z9eT;9kDtU4y3~;MEG7qt5%%vI&8)VLFT2On;q~vvNRiXd_%p360+aBOQ~CqDUt{FN zacDhgTrQnN=Qxw0LekCn%eh1irJ^`c8v&wb1ecnbKwd-VWk{SE zyMy(#tRGXTM{HQH2O9|>U^hIxyV8d4tR%xYR9og`?e-yxLI5cu$xoCYg7#uX{y8xF zTL-=_&)Ol}$@8s_gNdj}cGJP__q$nfODXfk66#~lH&zv2me14LLnRT-ELkXGS9nmN z-u*wEGkvYu{pes>NeBJTl#hQlA8MR0`p+PfQ!=KZIQO~ny{u>2T+kyH?JUE@0|Gf z43vN@XjRXVoch&2t9WafR-(cZ#rs@Y(^@!so#B)5<@g$R{muLT1&iFM%H~;|#M$^i z8RFvCz1J^aTJ{zla*a;GsM6WxC)Dfc6@eA;%JAi@R^Wj)B?_o5H!Zq_F*M^)LgiE< zRZupHJZ5Qv5VmZlKAM4>i6!OwNvZgbBPnTRXHn!Te1)8&5rl{-)r&!c33hVpPa%Us z=}Vgc3IBX00j~6%w&(~`3A6Kl*CT;`DkyaKVzKp@z4iWN`zwET!zV{!1l$%+L^5bo zrIU2-W#-CGVrs*Qgw)ZliN{*R7tB^}y7VGByj}W{t7jjr5{8%_O*X@8K zPl%fsAr|g$J!(t23;$X3cc%CL0-Dil6haH?6b@PNNfyjN{LVcPgALN$AMSNl*%Zc00LQ8<*cV3B0{pt5+LV?)MO@y6uIaUXw}>7Dlk<0_X@y zUP7@!Q1G52aR?ez#alxc_mb#-oQQ^Mza>)i}ztOQ)J8fkAm^W zH!;;Rhfe(2-_IC?cKj-S;~ENhuW6@P>m*#tX3#29d$CPlr`1Xpl`MMnZo4w^C=?Ps z2)|tnVr@j!>7IlNBfh5dO!#*HxY=e4Ny!#R9seNiQtJBKTL7D~fca2IIDsWRpQ_X< z9?DTy_z|xAUabJGNWR;+%w%#Qv($ADP3_QZB$QXZzivhprPZcza;o9yD*1<-eiIv;-UsD77>A^+F zYz^`RDa=??YfZJ4xbMd{!`6~jU_-*NfjdgjTRZ7;2@M2K^`$1B#M3-Da`K94&w`q; zSORoAnK#Dzy(Ud-nQbE%zFiPkKRg0-crmdcqrQ$`>UopZ*%5K+$UcdNUS(YfZpv>? z_pgwyycysYB&8vnL|F8L?m4F4belJrA_&23J2-VXtH4p0S&m`AkfN`7Zg8YjF9`7q zM{F;TB?wGFdOCYEkmHcpytM?CB7u9N3Ev44XFsr?_f`&)+8Y}sQC}mRX$eM1Rl3h5 zua%R)P@7aj9R0`VShnEykXdpx3>&oIpz&Bo%j(IWG`ESFE6`E9X63QGt+^P`lG=g5 zqFRAG!RVS<6O2doc+Z*e!1Nhb>#{M_=yphZ=5etjbH}sDN&GW+zJYy3>RS-1&kh{F zBY>z=1DdJh0a4;Mrv_gZH(cIjq`!&`(hi3zBoQRIbn5$Ky%v{(DogGEgqR zfJ4G&?H1{nD=8eR6QArvJ5bLc1Wni(j4KVNOzntyQxSeQ7ws`>!EKxJ=|(!ZZ6n6T z2Dh82DH{Nv0%M7xKxe6h+w78c&kLbGXp_IXXRH2NkEelA4aG>cVRy7X+-l?k2cAZ! zl3}XRUcn@dDIT7pZuP{-H5v3L&?KJ!;MT#9Yq6BcSVm7QHjTxrYcarFifU+TEP?UT zLboPpZm_%+7cKONe;P04lC3};SVw4L0hO|xtHL~**l%hs&@oAFFnRQU<#KEcsyB@w6aDp|l zD)K~Gf;eRTwUe_8%gN91O))H%myTHpVA5N*x*)N{YF0Wf*2+WL7>Z+U_=|F^pZ*5`9PuUMfN zMxKOi9o}d*JKO4vI%0a(X-MZ(5lxZSMhIP4t(Q!C%a`*{DW%%uiZC+4`r$EeI0B5$ zEUsS-$dD1w9&9SU)XLKcNW4JrT9^SdDPd&lZrx9mQ32K>Jy)N546-crB?Pj}r_j=Z zi_FD=QJcKGKb$lj2O3%}4{8{cV(8e)j1U%^v~p@N5bJ1S9o{wzhYKq8G99yIVb&_S zGakNQS!RDR>a}XQx}%gaDa_z_xn2RvY`gyI$*Sk`DNv)Q{(S2KMb~-sVan>R5Chz@ za{WG}dCPWbA#DD#SWLE;;AzlQJznZFcq3uny{U9U$uX>>76$r!_`(-PY1RD8R96Fu zfWl1wEqCOWp%Bq#$EwaZIFax#uURgO{EYf*lpK{pM@_d9zkbrJc z8nyWum4UX|wUF$%h}kkdImI=;>OxQE$^Bmdy#(}2VbFFRLgyfi(vD*JzU_aaQIPEKpSl@lsY zorqVcScCUzib{!WAL#F_lQr2%>yDe{A&g>SgLIcsimmWl5FG34dyp)lt(0iKPJErZ z5F}>~-@c_J{CX-(=A+GO3Bvw9(Mv z)Ctj13ps|GNb*=h>uL9?-uT!>M}JQ;Qt~`q+ihAPk4A{=+KrGF-@ujAkIk^GFdY9^3gs~FyvD!_GR)@wzYRnk;!dCUF zj(_z5j!pJ0RuucpZHuK1_OH8msH@#AQ=wUK7VMI-1KO~>&qkdYM9gK8m020x)=ICw z8T*!XhguA?U>ug;xFoHj>t1E&I5txA5p_!JGN#jjLgfFh*mF*%Q?XJr60N3YX~*T_ z+9k$~U2OzT-;#__a`G;7vZM@T5}G@9{-wyWd(f0T(i$>1KiMMu6KHDbZ5^v^IF}FI z3kCSQtel?}(oBcFt{C0G8?TlSDm@nk0jiWKmRjc``0^ZJ3`BWJG0d~xD-yJt`cb|b z&U5@5W6Sl{dH=7Eg?8nZTEnZus{q;b#hE!5-Ffz55ve8@e5!#C1&W9psZ|CBWa-PJNGw)@wZK zI-)2D>_Wd>$d#P)dSEz+uXa8CmnrL}%8j_@<_S?POrz{-tV|#Re~E0%m-R~ZwVsW~ zL>{0+{$S(qm|HRn)%PSWtFtk2>5 zLmNF#>BhNi@nz?rs>`3cNgf%xDBnSQWbgcFbvHZTj+Z7T(AWelXQ4RxNeoK`_(&fW z!{~oF0_$S`RNV)qFmlUTz71ST1UVQ_0XwBIgT-85CQD^Qc$SmkmyrU*psE7$AgH!8 z%KF(WHZ+CSV*yBbo!Un@^m>gbm7%v0&T_#69@CU{HL=OUymCQcjLr!rDg6zUIGLORD{cw=V{R&+E*|A;;|B z(bl?g`f8a^`F(rj2%>JrMwVtg4gdpPCLt~$T}1wbnpsSXiuKu8_EVK)!meJ>7gcRW zE%Uhu9FWG^RDnFH=@~|)dmM~D!Et$#VX-kn#RUiL47sD;;-~~f0YRkBQ)dC1Uzj%XLPLZaSopvdRG!^*a6VO`z z1x)4&qMRy_yUm&-MKgQ}WV|*VE>1ER_j<@Y7LjUYo5T8O zBD)zwtQVi83QB4M-uBK`Gz)?xl6mTOQup;Z!}YG?D`FrxsX~;NB&xN|gTrd65;R2QsE;^Bl90I@8n<*IdFi z@_MPwZ?g`D)L101Yr>YC&at+tZIdhQ2sZBZCpFiodXQyboX?MxmW2J9=xFQPiky;S z{N-D<5xXroUk+Yg&bQN!7&z2&# zqU28H9O=)8N&7G;4%BUHz`1Ymst02NXX$-hyTYV*#(J0?{>$Q4p~@cbcg^YhI_0mj z*vwMp2YGF~R9XZnZVoaS!C`YsMe%(x&Uc$@C}{rm`Mp`xcnquP;kj`lS( zgUTB|Cz9(yRo)?ovV6HnkLQ;yoLwZJxW~;6vj~%vTMdw0^qX!dH=a!&I`ZeCj6(E) zwN9DfhQ`h*LM=}mz!SOeqDDfY>V?(@P6sg8^*=m<@#KUqd8xLl|BI}9imq%47)2f1 zw%M`mbZpznif!Ar-LY+_W7|$TcGB_5-upl2-gECuxoWw6J}3M3^js!+DJ|DHnEGIslXNE-3|T>;?k&PL#W^t*loxQab@ z3ZZT$j7k&V98Wt2P(F8Ir@ov9%UqhPL3%i$SU_zkbv_E>9!11uNHK{IsU2gEV!bAH zbmvr$f`|-nSbVcLwAUexiA619^WIMt^fSDm;la!4^qDB+iEJ=)4lr7!WCU4suC{J) zy`O0_ASLyy4LVgwx?}bvJ%MekYScjahZ=vo(rv>pC`{A(4%c^Di6r_o@J}lDR2NFI z^n^K{z}!UMgRGR8KB9)g1YQdh(Wj&n zPx}~P|37$qEfFo_4@kA-B2Ihq&CW&EdaiK_cfN{V@_q9pZ0cmy{%0Q%uOz4NQi?P3|r~6Ge1Abm%}a*^NI3%=CfA2UFHz}+9X^{{7YU)jm-%yaswrG)B!f%<>{I#&G=*^Vb0 ziqxjNl2EGTgBi9$$$EqR{~&>xN7NRBd`u3BVTzxm<7a9UQuM7mL^o<8tag|euVpim zGm7>#vvYgj2WF-QP>u)3oqbr@@Js%K1MaLrt@v4+vH5Z)&qb8U6d7AV+2fADBx95N zYT7bi=#sloLWr{!XX6ld2y%%E6OA7-8%4l3xPEd_$oIUjhGK>%Xuy{9uk#i`tFA%( z!P*Zge=C=49~%_Hv@2^sW>Qpm!OH=rz|IV<6UEj|S|IPw(82+5eqWxFVnO=cV(@f{ zB`(@Wph5^*y+?f8@L>{6RaL&#`X%6Xy^q&!a|!)LL$y^cpICI|F*HVTZ)zu%@qLQL zjL42e%X7^Lk8si-TT!w;r2OJ1&#&iNdj!G2Eizy!IoY%F?IM_M=>szI=lW5LcAL@oOZO> zM|otrPndXFOQ#B6UCux{b1L=rl;}z~!DjE2sfiWw1EDT`M~Mo-AuZ_!$j!U!~0R}tUffUty(`yX&n^8kC)zom9&Msi?MjsJDb zLv|YNeN~Zfgx=bjn?dk?i|F_FgYbA&Ry+d6(e?G}VPV*fftc%unJOWkHu&C@;%0%( zNAQd{67~OpgXvM8OBtHR>gTRJKH?xNYuRpg2wDK_SI2a0J!_e z9UE8Yw{?4NYI+k#KBXlO(-!T(mimdb6N&z+q0Z+tSuAJ!*nF^iTma!~XLAjk~oM&MGUWzQ});x9au;9{zJuz+|h9&wkPV#STexniR?qynokE1Eny>X|;? zNS9rkItiskGvOVqkIHkeFa(ZH{l_R3Qk<6aa$hDx+b^^EU{eV!M*&GxzL{Ew8L@7) zGd8xfS3URi>PB7f=SX zxDt@3O3uSX3h>w)XBQRC$|7b8AY{zRzg<2ERDj!&8%y;wO1G6J7!;1O#SnD^8My zYIm|{GSs%`8Sq$@UnSCw>iuxGWr0F?!wr^q&QSd{6lk`!lorzp!`F(%&pDD!W`(lI8^_EPo^__uuK zTO0hBapC=c+AjHiP8COti9g~#DT3=e#1c>`W2e=-AV9e0YUIW1v@rX|Z*~cZU*ijc zw(j2PRGpOGvbmVS!Iu_oc7mx8FZ>?8)_k6WnH|A0!usFT;O!r35Q^h)$ z>zf+vFi(V$hnP2Z0}>IHzNta$H#NZirUtFDw@gO!@#Xy6aD*0gcn;^o8BB$%4K4}# zHPR)^%(%S9)%TC7u(|5(Yc((x2V0AuPbJ>mj75f$SQqJ`Q1aM)97?(Mi)QRKr|>Gm zm5ISjWwl2SGe~ngGv@@$t5-9jYU2wmzF)kYmBbEasu;Jt59f#{415GqA;twdo1<`M z{`=C$%-fJ6G06Apu5_e=&2+3Zf7pyjc=o8;#yW#3^%-H1!ga}*rI4F~u-J$pTQXg% zp}gixX#SxFnAkR!b^Xzw`03*lY|=FX8sF65E)ue~O;0__gXEslENSmc+r;sftW>IT zUIIQ9Fohx#61BuDA2)~zm74GbtLYJOVz5&O{n_$q;NI*3TR#R@Tl8S?7IFAL(7@;) zXfW~(4RZg122KA14PJeFu@Y#0KV$jm$KG(oenSJ#Z)ouLf{`(c$*gN>`5m8LuB|03_bQl-V zk89eEdkIMv5U!ns@{D_aFXE?XziRy9lb7=jadd3)6lv=0 zgzV~RBQ$o|)~di@BMw=6dQ<)9B4oMxBs%s<7+f9tiMgeLGD0+EJ8CTcLYM1z{7HS( zt(%ARp%RNwY@7YL!v*w0%&?(*(mI_@kN49e@~g`fxWTT1rGOfi^o5}rmb8hcsf{vR zCSd=PaRz5}959>b`K+9Pl2udB_7cSEw-yrNxY#mWw$6mx@JFWyPYv(M^a8t@NQ{%t zN|(tSKh9C(6E=ziId;rX23gGg;CJCYk5 z1|=X>HPuGp%bL8_P!6Jt*_-H$w@{5LaivqtnOZ7>V~FUd_Q6J_Hb`Nlkud7|xQUXt zX~~fNL;Tig)Y=UXNOD?(%wLj?c(y6aO?Rmpkyl=J0ZL4nq8F6f8<|LSxO5}qiyDK7 zf-IBh$O@?Eg-vx_&6JwF`*p$f#i*k-NfXVqe)v@^xa4-d9iI2;y&I%j1_Vb{53Ki} z)OPNX%j&Jc3gh zMa74fBnxI%4Lp|}4X;^Fp(yV{A6b4KL@zMuXMF^Tp-oZ@hVB>@j>a{DrRTwnqQzcP z9UG0bWY@CnZztS_&{<-Opeb0*2SUQd!K6OB$sUusmK7g>*_8DUE)eJvFlk0Nc%4kV zK%4$LLjSc}&2GYTN!NcJlD(`-O-ypL)EOmjC-?RAD*Kh{C7aY7he!hjo(s}tBx{FN zP>k}Pm+}~fzr+gpEe)_bg~nhbXJdc7gXphnfRg`p8U+#VDegy_=vY9cH8kUKr)~BxaJ}pIdc~Aj zScpz?lQwFKyFKP3jbmnTAO~86FaG)UrdHw#vC9mSMZLW#m8Q~&E-x`Xj!p``G?sbj zEP|q$s>(!Ek&&P%dvZvlnYfwb{9Kts%8HZ)xuslH?cRK=MGcB=#!JSc_<8G=*&xF(PQY{+fJ_iB{DlF=x0cgpv-EH z*)L?}LGI$;*(GqB;*1b~S?kd|+0kj5y8g1H*L;+H1X+Za|9{Emdwx0{TntV(FFSTN?2E zrM1Cw3(%gSUu>3vQz_UFG-1%OEDy#4u0 zmVdi_T+29|iir9(?qPh7aP?my&&>BwH}6kFS=991=l^y49^l`8|BlstkQ1O?Z4&u4 zG;R7jNmiwJ5>h)+dQDyucxTyDnDOpWPpF-+%l7r#8VIIePR1AWG-TB`)SvfvItT@Z z`DXj^!O)%4&8X@;vNSX!G*K~6FJ|j-#v5+loPkS5o&_J$rXBrnYk;%OhadFiztc{S zyVgwA0Ka`ITOo4zw|yg$db3(vPP|5&zuZm9uWo;z0j=P~E`Dz%v;6{47l%3u=$&Ie zktEFOQzpceeO5aD`AaiOb62JSt6y~z*xS8Jn`Ah#whHeJjAnf5tbVe~(i-s}YcL+> zV{Oy=CGAd#B8oF20EW4w%i|z-Z5RP0Ke6c(8Z6J~Jti?o1QDJ7O~~K1piqL%hCLPF z0)e~Mp)&7>=$gb@_UF+!<0(fg#bxo}0rioB8B#^@ov>0z{wTzo}H~QTJ zpq|Z79wC)u5@ux5;pxFiW&SuCs~zc-V44|f*+J?w-}29J`kmAm7e|cPg=8)A#oa78 z0PmgPcTR7ao-ng{<5g&hp-Wb-;5MhSEDT#I2$k8|ig}87n6ZnfP_{0V475aN#UZ7|;v2Zt`V0DC=&3?HHeLl~P z?0oR@e4fwEA<6^%K5yN`2~Tba_3^$Q{{o)lcJK)a@jq^+HfkAW^uI>q{62+LUf4d* zPl^av&k0E*@Cd&id*NNrftMSeWIrI(CSdOw0i307`P6edYQn~hygGjZlJ@Ampj2N} zYphVb>Z~uOn&kaR-xhx}U0tYDF6_9Ka@&4ugDnVk(PJIlt+8;F?VrrnQXAjeVC`EQ zRMeL_={7ehDu({64QTAY4_e5lV$M#;fQ?uVi& z*vJ}t8M4x|MwY+ZDhP%qD@u;oPpz5`ted-n3 z7Rzb`Yx*#bTa{Wit;DjUGIEw5Pdxpd(?BSXKOXtTifV;pI6*G`5M_AV9EHK%%Z91J z-d)Z|0*)4sn2cYlDJIoY!F)!;&=MxA%XFuQ<x)&K-kR!>=E6|#Jt}9L2(dz(jjW-j2IcE=GSd^;c z>0YYj;u>T;NqWNFYp;ep=TgeHvUBr=m=4(5kEe(ND_d=ghqDiiXH(T9lW4JcGOR^r zrh24GRUu!=N>eg-T{$1JbNYyS5~u6AhCL4HOc~<`N>3pek{Y^sfxG z>{@bXusI17u{#i;@fBgAXc9lg!{g(x4@;$+SJCC*;Lbv4wxPeFw6)xozmO9cRe@Mp z@ffpn|E+e>EM*RXBHC5?rzs4iN{X7rl%9>06(06%8Xwm67y1!Mfi#S`M#0R=s2w=n z6b$FrC6Vr~FDx}n);#pa*3(4EE6j^z-G+P#`Oj2r-H@QEYE7%oNzDVLtE}XBfw}PA zV2~s`ytB`KKcQUw!R?aB#fPw%#F@Cg(OzD@FXvl*%se^$Zay#ZZOJK}@+qx#3Cp*u zco~!2btc9%ON4*00cLrZYETW_A(@%bDo~=3Ff0r?Dk#jhTvg{6v5t(zD;b?>0RA^N z$fb6)Pp)l&f$Y$s)_61RtXGu;9|}UyzFa@IIA@#&onX441$er1g?ii9TRL;o%fGS8 zYOZJI#ZUYpkYr-O_m}~eoZivT{+MB+L+zw~2E<3turl&EEq3|Vt%5I1^7Be|^l0vb zIti#0#_TT761B^$WoL_n8#ebofL|mc-?z`=sbI)&Lj_L)9SXumi~{teGs_9+QH?X| zR#yAM&p+&>*9(o!yZ;-o8g#qA zE6(b9*_VSxipOW&3ik2N>ch}51xfv=&Xz0oF#SS6K2OI%8!m}vKGWA?jIea`z@|Xp z0uh)|@pwh$|Gk=v zSQ}6nkwj{D8agX8hLy3{;nAwfRQZ*WhpALsgf3uoso%*e-(@~Y!5JE*Sj_fUB`T4e z)|3Q#-2X8Z3j<=rF6{#XZ=l7KGK4FA4)=WWXXf(oSGNeXoMYuJ955eBUHJf?26f#q ztx(Ru(rbxU-Q*T^2$7orLxJPZgKZqN z?c@wVMp5j?aH8n(um^Qa1=kruT^>6E>rf09wl`c?qilGp9tzIVsERJd`WGto0ubeD zkBkm1U0zm!r61OnCE44iS;C-MHeZ8$#w?uu$Y7LY3#Rki0R|2TaHV0nO2^q*K5xd@=ASjI>FBPC^ie?G-4}{4p@j^|tDwq3#|H&1|}KVjMBn@?vaT?6%O8>29ciVK~Mkv4{t@2rm%T!W8!I=w{gQ$y&mDy}4(COs>v_Ke8q z;N_$n7qx3p>+2Q(`*qnhU(c&=)>P3nUW`8kSoO^u5G+%^I)~E^WYLs*ZUax%Q(DX^GMfeFz~KS`xqR?YwLi z8CE}w*t%e031nns$JE4M1k0?Mb+JTnW-b>8h7DjN8`v~bRyLOhU5;Ut?rzO8b1NTc zZ|*`6^G^sT`Q&H4*K!F+BNhgp07&uW?YFrbmH!L4)3AKZ9AP zPD*$zg`kyUIh_&6Lw$AalGxr)$htW0(M96JoE4LdDT!0^Oa;Y3Hmg$^dUpREh(4U0 ztW;Kxv=|cCNKETt;@SE%XCGrc&vIZ&DtJd(#~!VsI!P)bdgPP$NwUj4aL)b)veu@_ zsTlDjHh2B<@14NHQ;<{U9U0WJ&B)`}W;Bo2ZcsLoB|mtc&*3vFE@V_)16jCL*G|2x zDjHZ+*VcUODjE*>lvD8CzuV}}sF;@ahAF4yl#fqOe^-C^fcvie&c^zD*MFxz-M-_0 z-Y9nGzH?iE|3>lnE(D#ilkfWO;QU{QX%*A|D*t;b_cnI4m6Qj=Z(TT*aQIIsuAK+`sDRDpjV6ykP|AiYEA7Kn-d1fl zKOebkYr>$78dNgQ6WL8>SOn#2{KPJujtp!5Cb|s2{kH4jcBox~rdxjxEbWzd2lM;M zjLD$Wq}ovf*}N(&tb+Xfcr{N;uamBQ84d%>z5_xnJ?bB6;2MqTaURYgI)zU>4_sm1 zp_!abHkjseD-L*xzf^FBzaD){qT8pIg;TdV?~HK0H*#h^i-T6s&VAxW5mUohl- zzQmc-GqqYV{3KSxd$n44t8t4!6JB*hx8Ln@QD+}&Kpgc$r1DnX3@{k28hLqq>YQ6zfbCM; z+7=h{%Sktx9b-!i79Q~&@KY}DBhqXwkYXk6E=H&PdEZ5IykYA(8mFeGX6O+~#do1v zrlx9TIpl|%EY&gIhO|jMVk_j-m3vVv%z%uaBKY0LC1Ai6G|R&0^I<|@kBPW}%+C|% zw(JZH8|6mD$?%wUB03eFwZRe^cBi@iy1Uk86C)Obnt|bKVl?JMN0BC#PM%SuaCs47 z4~4sF^P3vD_Mcw)^lpw;jjz$T4jM}st@m2m&_;~Go-J|XJ~$z+$;t{4+rje zRRmg!7YOK^8kEeKS_%Y;L@`$43v21%`LeIA(Y(_}vPT>iH#(&CPg6 zHi+G#Yq3*DNBlv-iUZ;*5T#BXYn$gMtI%yCy%!_M=b5#k$8NG_8j_vqT{v%n7 z^u7Pu;1dC^t&UY=9S8`B44nITQ?#Rkzh4VHcyk`U;8&F>w>YejO^rwA(y5zLT}P^s zzh&n-1(s1e(aiDpMaj?DF{uKE5mA*;p)F&U$s#D9Bu2-s856tifxCtv>(DXZ+|s!n=N8t^p22rnMhP zP=xK9u77KoCb7LxS#)+UM; z14Hl(zeY1=!3GR4vG30pK3j{f)bYF z7Ep2d9dV7EwP64^!O!^-*I4*d+)NSs6Jku|?`W>nMi}iJ!oKC$;dex@% zM^2+-Nn#?#a;n;a0$W0h1eB!*n$M%Y6hvE)yQ4)T)k+mmsm!#a##1R7K!u8gj2-f4 z+e}Al#q!gPbOuYeSqjP5i@@mA{Y%=H&u0^*P%*Kjfm=apS6pu8jGT!So266 zL696M`SObjD+4n-r5L{;rPz;s?lS{{)|H|GCE= zCqvnmDIEjnbPi(t``wa1xv29$Z|6#&1_+;c#s9R_P?j6>Zu7rxQbI=~14Uq_<0$;V zdGM?~Pd_)1T(UsQNFEM!%`7p`Oa*cdYB%)N{M&dha^$;%e-h7|HMXSA2|sw?PO9$O zV_-2UfDF3&K7pG8;L^KWsxryA7;9ajQZ;FocIw}kx?b_PX; z*EYc1?zs>X)y0Bn4AE0+SYoYssn@{*_3`KLJsq7Jq<~1}N4jMF(KsF2Bn_^@3NFDy zX1Tf^vV{5*-kf;$$8AMf>GhBNhgBjATrp4?45@&1=RokrrQLPxhd=psJFW_kWhpZ1 zzs$2aU9xwdOYJ1aBBO1?A=#BNi!Kr^a2N};FfCr7zy2oUwXgaxOvc<0Pmg|TDpE1s zj+DCe@pE}TK_)sWT9j}*ckBt`Y45&)zZ}qS;O+ZxVG?4=zM)F42c|`7q3{qZNuPog zm+j52g_3pGp@oilkPFGa<@1b7k-gx*b3x!GL4 zP;>n-Im0ArA#|=c7Y|hUnVjtZ8d^s>dtyuqu}th4I9sLBNyJz5*EQZ+xMj5Q<^F`7 zV1MoXCC&BD739qTRL{I(tzn!#-FHK;jvjd`?hkGVscd0YqgqLVmQa-r2~a+Ddb8?X~(`R7){Gm-|u%$2#kk3NGeGe zNA%L$Rbv3CKCOLF`^;p|%&%Ro)Nz1HeC?Wy=t^uLU3UmLv2y#F1HZvMw}frX!7W8ie#)8BwA)F`j-nJ0M2lv2$R z9oN?t`5in~6qmhf957h-UJWk~)(Ri}W{vP4%%@UKh-&98%3tjlyHe68BA6q~Y`1TE zyIp|$6TcxW0RcfgiS)eVLzytUP?oo=`-N}gUso)y~_8KO%9>AmUDW;?HT zvsSSIorOXJ4SL|^I{h{x`p;!tlG~OA@`wQG$do7CU8NA8_=~|G-Cr4l&!rtBHD;fz97_`b;x4F&W z@15{_@(o2~gUV{5-MiL6U`i<`)WzxUDpspq?}*8P>()EBtco`6BYKBI&@Y`O{AYspLjnPd_33`T?1YzR=;O*~C(#1%7d-|Z+qDhZtq z%`yPRo3hXLQ>)s{mlP@HpI7 z5NZN~Rhtc-8q?H0?RwtKfQNI;(-d-hUxY5m(3`MqrjRZg**T}T9Eaod3RhZ&G5A>}r7 z#b-iWG1vhh2sL0dI`OkHhZCOO+i-6O|0xO%=uk0ATSG-*9)YbfB8T}#qU?NF~26cnG& zneY(YEhU5Dxb*rU$HVUhPeDt{$jYORoV)@*P$s6doS)Av=&_-&>QpDO5K^j-C_}XF zNx)PDg$xp|QV={zl~1P!Yl zO(i*USNB3aFlxe8dG9unwTgLr9)f=?K|QC{VbeQCyVG{}3TcE2tY@1FFneTHvQz zLN?JsnNkcnI6}&su2M;b-~TPq9N%R-?{~%yoHFAe=S5JQY-3Q->Q0 z`0pSb^I%%%DZ2e#kAX_Oc&P`MN?9;6drnGynU=IJZjms&+>Kgs1E*=n#-8dihqhS9 z@Eo6Ah9>gfshi)W$1L#@^9hCXn4VJ*scYN{%xtL9n^c)@M90Mlf}hl z-}lE+A%<(MLG7wopY9uq$;az6fk+AVvRs>nv~1rx?y_FkMwLf%kmb}zKjKJmg%krq zX>E$gPkId{S9+Wvo5#+S3wnwKuKE5|!sTm53O_<-n727`pm4*B*ji@;w$-Ii*zKSfff(=|%< zNjltZDDPZaSa3lSM|rPBnCk?a+jMb$`|R1^xu}8;#x)Pip8YPimH8)F1t@Wh7TZQ& z?)+Q8zXUI{6$w+qYu5`e;*XTjbo;b2(EeqXCZjR3Ds-qyH))(`B8~kgPV)Q+8d>bV z=3aGF$xR)yWqSkjFqJNp@)U_Cd2W-t+47H5I0!PeCN>-FjrDn^Df@i86&Yl!wL{%G zHWrqFq9gR9BTI$Kj`_Q4^L5gYw_ur)H+k0$e8!U)=u;B|rghcKp1pL;<)e^bgJ4mW z>!e2XE6xM|1CyS*XV1o4U8h_UKjSHA692A1694R*%qvjFLqPpU)%iCkq+^72+CoN zcRdpa(%x?_E`%pTxP;VHH-WbODPZ=9j2;Hu8FJ=L=I-kAlnJ{yQp^iG&N+NN8FFRJ zaDRHongZza*EibMrhb|2r0Ik`RSGoAdH$}2HacrDGl>qWT$&6k5y1qPR`{~dja<&Q zihy^^^&RqP{N>iQ#BM$7(&_g!*g#wTpq2VwDs*`EpY5Q!5v} z>GadDLdrE$&!x=9q0ub2*d(4-^sQzmwp~Oy2tAJu38LabZqTKt((o1{@nQ29C{M*H zoLPsur_E>1gJTer2)SVh08o}z7Ru&N$0$Qr-2;ldUf2LRO2X+6&eh~xqbYcUuSm-Y zhqIkA4uCYL`D>dS*Zk?A&EAQ|HCq?5dXh8pq(I!Il-m4|`#!yesco}b9TL3{y8RS0Y5M%&iF#| za!RU9ot(4i9s%q^j7WbPKD$v=_LKHF<@5(9q#bv*(>=zBRf_yQ>e4dd<-)5Oi>~d_ zaS}o_r1~3HOZ+)%9udSQ+41Z2I3pp{`)_l}+)XEETNr`EISChN$(+7a^aa|Wx!(#( ziGym7YE^bSV|hP%T@zI6_x<$R80OE_9vIoUk&Bt;|I3w;fZ8SY1!wO zB6i&Az3#`K+jy_bKR|&iISbxl^?t=$@2i+yhZ-Qm@=M%XT4CzL+slsNe=S05?C{2Q zcp=nY0BUQp2?ypJlxXn^^NU?F1#_O&Yr;W0Xg90R8Ki)CiaoSoQBfqF=#e}>z&%{ z`KXFBn}H@spgRJ}@^Ww_uL`8ZkYf}%##XuFRC&g_Rc6W@xEM6DUnRxL^2zQ(DUd6u zBGkplx>c3TgP}d+CdPc1b>#j+IzFB0Ke_(+K8X#DVf5>B@AP8E`Dx+w!f)zW;Pa~2 z$E;zi%7qW1=MTo0VWQDZiWYhFg2UIb-cCci-FX)k^3#)#1Y6nVl(wZ!QT>zj@LR)A zOqFtN&EJ)cVd_^&LjA+2G9VBZYR?4|pS;e_6rc|#{=Td4vJii^TL{L0bg9)sT`w_u z9r7+>b~po3d+TS0>?nK}*a~-ed!b=NcJT*@BAYuEFo>?%J@uxdY(pfoNVL~VEx*5>R4OjKBHU-L;T zyas)I+}*ucOEyDJOmCPalZNaMcS$eY4zm8!eiMw0E zT$qJg?GL`0bH0)b&HlI%!p!$=hy|^SCz<^ADs@%|>g9WQzmEj!*Dd9$u!Y=M=Ga2wo75c%R?GE>5elO zlMyJX62v*-e5bWe8b(0Jq2vNj_zW7~BXK6c4eS^KotflSz!^PrFB^JlJZo>G+eyR9 zLprrCC1jf2rwEr|J^hiGPGo*=Aulf0*PZ+AKCYK%8}F<6^IH45SNXbkW!vv$+mGh! z`s{D*hzNVR&k!w~WRcCXQ@g50;jqZ3W_@H1QcaF(%NM`kg-egMupyI{s7AJ9+gpK2 zv!)Oj)YzdV8yTjfkVytfmi!CKPm?~$u-Ni9ryPJ`4ND!6c~E4Q{!IrzqQ>|Y=E}>? zGuBGGG2nhs^a*ntd(YzXsR2U9vYQLyFHxUjdn$sIa+2m?P0P)dN+8xjG>+2!uh0mv~Pw-T#cnF`RhFp2xX7FRt?-8vF4aiQ7 zsgvJQB}kzWeDDIwP(eQYsM}~uqSk$@MPN}OCc-92NngfyelnkNe%N1-Kk3-YFccP3<&5a?PMC6gDs4Jt#V#qeIRyx-C0IQ3Mud`k$;~62=w+K z)vH&XCQNTjaRk}Ti-rt2H=bq{d?iI;13z9{qB-opn@|#NUsT<<5^krfYBT3l1)cRg zf@gRcn6r}{{2?(Ti{3^K4~UBN%UWXb)JHW}9Pga!i)>Om=sxN>#OpbGE(528qD>;* zME=sh=TA*i5mg`PAnJf`=ssV$2@Pu_#89C;hozWTk1oN+yalHA+xVOZ&pUpIJyEs2 zOkn@=wbu@cyGwPH*w<5;Ag3YBhRv7_q%dYK)>}v1WXvoaMGz8{-e~XdN2_WwO~JIb zILy)(OD_3EVL=@=0F&$C?yqCsf@4lw>gHE8)~rJ>ZW7Qg--20b(y|L{vL+ofEZVrOMtlD?p>InYR;S!mlxI%c-^pd@ zlH=~bf^1%cP6T#ne}Sx&(A}r8dxh$n81lZ>>U7}K`KcjRz(QoZJOS>H+#^i;4Gurk zAz`Fbqt6uh!O^M5UM#C68%m6Z@%W)bw|N)m2++dqS<`p{4QDwsEswxEA^285ZJUqe zM%uAKCWw1$#QEHV;TpYtM(#+8K5hkU@~*6)xRiYBpq$W%ew48Be89dT1ay|w2;{Zz z!s1#fi+~765qPUG$ckzYKlcvEdYHz1pU`|3yADJuXef(nQK^y%1ZK#|g2}ru zxcykrZC&z+zw{pZkwZhLE^_8!ZwQ7&a-KQ={c4+ds5QFrNv!-+MM(O{Z?(SRS=`nA zwGMl~GK=A$R=GGvyM#yXzAqi!3%#IX*lBS$#)C9WR*4F>aR zrCp&^1rD8l5%k2wY6yEoH|`f;y=N{m6d;K9(XuxFdPrL~XQm8D;p3^!mPk$50R~NF z`ao+Y+hAtn(xY(V{kPs5ZlVc1Cw<*EhL&rZbC9j`6qnoAqp+2Hmhvc1m_xyu}M(X2;o9qo65p}q=zYU@waLZoI7M< z=xRQ(XCxm2ukyt$+U5!0Kap40y-fW`kVwwz!V}9d!P|^oG{^8T!Gfau&WTEG4(i1b zwS?Vt!U{c}5vR7`1MbZAK8UUc_BEqVL|b|;TBHkH0Tp&+V>v4xPEm{5TWugCbX8Ix z_ZC!ApA$NZq zW+b`l7=4Wga01n>ZKb+Sfc99b{_{`GT`eI=Hd}Gs#wn3fd>T_oH_r{vVj^^B50`Hwp1bVrXEjJTYX`B=ohr2=(e!rU-k+eJ zfNh+{t~f)$iU}OUD_X1~%P#U+)RskS;;34vU6A0Bqbf-J&v! zev@_Ud!u1+@?_J_VG}$pv8w_T9Cxdgudw-Vsrz1ft+gq7^m6=tAX(hrxI95dOGp-L zDWsLjx|fBdXv>m^q~#%-k8^>G3*P~OXH1I7&IKwJfBa*ua+**;)F7mu?Na$5NTXI_ zV2ZTnv5C+Waq_^hRy|ME^VIMU=L{dT!?TS-15;u2&cL7To}_l?ej?iA4mBDDBg0Z( z1j6C9dMOR=A2X3=Edh2|m0w%7M1n4@@rx|DMHlFp@BS>@I=YNHJkxdi>BAWbh>CW) zENWnSFBI=mTH~`cwF9+Pf3>m_dX<}falK*X(-GdUyqop8-C<- zG-d2-zk@lmd+Nf@nH8;M23C#tQM;sJ^{F5D-0PXR9N`s}Eweo&I3W}42dh6}JyJ66 z8gp_j#p^wfFp!!tC5MXNP+&1d5Uy5?n@g_hDl(f6c?zp1!~VigZB9tNa+XBJPmVwa z&Zg;#2n7`pN4o=-%VI%h1h$jrd;}n5bg|Ln3=c>)9$p=rQt!!WZ)EV`xz859E2c*P z+{g;R5AUj^G|@FAQnnRxGi)$dc@h>%HH4%qOu3Bi=r=j&Yp}G5VN2|=QRBI#*ic!! z$mw^1KuX~!r7qlg14%=jHCaG;uf{VFZdQ${xH}J|!Y=D1fNmT;pi|MQq$;%T=MO*G zxODo*Do*enZb<}1ZRhe!E8%)bu4 zC&GEY??&=2WBWH2+0F129&j5$h?6m^_ScE!KR{ik35;vPhXWAdpnt%-zpor2f`nmG zhrl?A-F7x~ITKYQ=Diy6-m1o;>JxHVoWqbfn`>o4SYq7k$M%b@IGt->6Ti(ytK|4| z2gN*b0MrsGA0BhoPFO1uZ)_JX`HpClxA)9aL-~&A2`NlLoQgZAyKD$FKmudz6oHX4 zbGCAiLbls{e4QU}cMtlPZd)2wv0(Vu_+3t8Kqu^a z&vw(aQPmGv+8BtQvni~i7(kWPOi<*J@zt7dWqHHW*{{m<5;n7MNDgSpqhdbLnTRtZ z6P4Vg&fvp`+GWrsrOaw-pR)NjatP#|KR)hr$Ida-r9&|t%wP39Zj2u};Ol^i=I^tu zj)Ff1%?vv)=9<;bo++?eZppBZe9{io43XSXt2|`S(g)xTAno@rCO)X&U2Seb$vo!K zfch(VajW1LqQ|OMjylv?nB1C(SZS6QQ(uL&vt_{`fj}s0yM%dmAvb~+Xp@38k1=F| zs>-J7d#zz$9Dcgzs^k2vZs$Syag#cNcE_LCVEwzq6YDX5vf@8KNM=m zW_stwjb zQ(Jd!_=oaVcP226rm9$PKNnI=6~e(ZrkmdN8S#H!ooIqSI6gQaeBBP;>c{!He*N9L zz5U*$ywmmZdiVYq_x|^UP&7sVpx2lGHK6;GDRK1nR{ss`ZrRzX0ML-V2c?Ik`?P!8 z?9*U(qt`D$uYrp{5vDL6CFRwsyrX3D{Xd1YE>>^7tRF4Ur(v(e&0>SQlWhHBvxnML z<-TmzcdGTV2bo&7ZN$>&>8wuEenm;#%ZmThRX&Lv-K6qq;dB@*d2?p3$r*u>sd|#} zzl1#xZ{#8MbnduO9E|(UN}p8AzEq>q8nd%Xu_nj(jS~BOLHBu@YG`Ik zHEcMMiW=_s*PE|!Uk6>-Iz2iZlDhpsC`>(Sd@C#`$)4xmj&TCiqaSubA)MV zO%)6ssD0EMKi*qHTApC`kdh9pYz{Z4JM=z_0B!78!CzPyr=83jWOLZDEfZz}eVMcJ zTcDmREfwr`%d_L)G=M*3BVKtDdquAWZ(YVFE^UEaku`#3vXj&7kc4`q@R?sTIB4CABeiWyU^j^Yg`;=mxiy%PC256Gk|&6eQB`4!>C zQG-wUm7q2COv_Q5qk3g3&F=m)pMYi|=FNp>*}LyBr3heqf6v#TWi#jj=TLY#Jt?qfT5+?yk#R#%8}{$A<;NrLC5m8N?)`xMQ)y5i6&I9euBJ1UgzQ4=He?iow-68L)@jfQ|8ls7Ww^(sRk)LjuJmvff`yYELe1LoZn2B7Csk(J%#aYO`UZFM zBOmywD&);ZzxPStWdVMh>ox8ruiU^Bc$-A~{{pi>Ouy~mi`QNV=NBz`tuC4iq-^9n z3fUcl>=5pTL^-P{w&l1EvO6CO*?jv^80Kj6HH|u&u(ue_>W1H$QjH%JGTnLE{Zz_D zE%{mmCTyf*hOatWFx$Zl>!ic$+l0+SIJv;s`4qR>pt+~M0U%_V300M>kwN-eIaF3N z?Vt#d4GN>hxpo3h$y6bQ;tQT}U7J8EW%OKWB&!>q_NFP*TeA9z7X?fE$BohFhUk@t zM7E)MHedbJHJ?yH7A~j-Thkkkx8N745oQvqS^5Pmk_+W7SuSrNM-8Q`H4N<0U#j{n_OwA8(pm<>OoLzfKFnrkAC!uq5 zf25c>S=1OG9b}IH>@AlH4pqje()NRrrN>SwtAt|u#BZQd=F02b&aQyD&0j{CtQuWq z7sB?>^QRr6b1#E$dp|E)i_w-&+%(lNF#-2`omgfTC1<+iiJFs(dW!4g&R&o&$kqFT ziMROHCA-s4=TGO)y3`%g@$$3z)A`dY^Fz}o7{u1!9N{mox;&(P5WX|AMYZ#tuKvfo z2z?Kp$WRt!U0XpIavo;g%%>uc#A?$EVg56?{N-00rMh z&kUZ^bD63+c}wjM1E$+r|uIg5opD>;Hi0gKapR|Z{G-)T&hw2&v4C>{G_dHM#f5r9Yr4*Xo1vB>B zZsMaN4002TaU|}NjTnaD9NinUts58-hpbaxW?mjt}=~gGk z`goV>>rz%dc?Vk{^9bWXGq_?cJV%Dmi57n>CY`)N&OiXkLZ)+aA#-LoA}o(WP#hTS zH`Rh=5S!FT5p(WAT)j1fjnXYG5Q!rSmR|F-MMq<%;XbMVqae?4hs#?C@cR1fRc(AT z-+pxg(?*`-TZ-uwPssvIj6F1sl7A%FEkuXaKzeS++itox>WbG2hl6(N~ADm{lbfXUX3@>L(I3OOt zk^7>-%^TSBJyiQng1_g7V266`6=fw(qdC6eEQaxEX}+_Cadf%C(m2kaP~TZaDHGO@ zlku%lk4hsz=FJ7hEjOBzw{$~jrlcD?7u)yi+|2d>bLE08q+ah_%VrsOFG|m_*GJQ> zX-8k6qJT1LVZu_QN7^7gs>Aq(_hEtl!aowPgRSvS!WoFz9A~?rGR;;1GpN}!AYck9 zW~J=i?3H3AT7v=DY3Bm60ql73@|><1q1x@FzFSPT+=pC~RDy4{?$)2w(oC@v!^D6`H39fNRNbXtR}hFk4FeFV_dX5H zzVD|TIt@b>yZ-oI*n!_F*5iWNE!ed;-t@|oufYx6I{Ucia;7O{ydlqWFZ9CA#jjxB zS`A!dkt^8IYTwR|;Af9?kv$SWQ|t{F)!lp8Ffoo~P-tJnM_9GtPFuAB^_tY`H9qoi zzf|CDk&~^>=q^Dx_Y~oi8KI=4B9(a$$?cw*^#Q1GZ+)gq^kJ81*uf2oePk8AH(&(d zs$M#@3ZScdOJ5#x#`$}v-~Shbm@J1+1G1b+?uEu26FwlsLx7JU>lWkjuIm8E7bP!+YF zH+av+Dh$vxoVlqeQf{PTjIIriX6Ogb9`XmdH4Quc<8QXDHR)=v}cI@QY`EMdtPq@^bTBe zky@urE z1_mg8D_}#JzRrw3woAdyOn3$Qcy! zTJu7Mi=k|3Fqzt@ME3=f zwY`D(CU^lo7jf+lP8}_by%=6rkZz#h5_oVb(L}X4Do5KAljE0tz9N9EuFYH-twgep z7NG{zyZtO?$6VGv6H2aGDOg4dTGE_pjME>i5cD|ZcL2WLlD0TyLl;_nXl-;iz5@f# z{`>KuYC5w)6Tkz*IqjAjXUF*-KXNxrQajZ5W6N;@lsTLr1o;vr0D$h+l$lohrpL?Q7Z<& z6Pj9o&UFP~fFvRUyL6gCQ24)p* zBPKiB`2g;P>{Lb{ur{~#hpgf2aGwbGFFOj*Ie1pM%h>04P|hi1dKg&UQntb)5{7=EU9QH?kT;Ht8p7(VRorcT$@ zz=--)nO9rkVQm-Iu+eDPxz*)ShzUAtp8MTwfy|)$+-{F#1s(X&X6&9OiL8Wc01} zb0*B3{;p05tWyH(?lUE@_C8YrE142l$M$bZV4V_J_m{xh2^$m2(7b4UH2{?3$!#5j zyf;uzjLXOKzH7~8)|&zSU# z{plHNTBbqd7r%OY3ugkpHfxh|Ws~I{-hIcQq|kD1g)PG@ctX_`$*W9r^zJVlEpd!CyLvf?vh-*hr(cV2Ba> zs7Q=R!gb9Ib1EvDWg7xQ>ztek(-GBuqIx=j%`px!s}}YW-Z6WOi^`?K(;Lb&y68Mt z0Dx0F5$nFr$)zO!tXR1rt4uCTPftMVySAJ)Buwa%W?oqDM?o&k*t#czt0r^K1)^H#y3<&$K7FETYL4=!PghQJ&6%dNK3Gj%HI@mUI<# zG2vTvXcero{o5`k&{*$Bie-r2%1Y>qYH?2WIwoKKRjs|fAPPStl`>uG(BmjsR=O(t z?t_wLj4C#m9^LQb1##pRkI6thd~M(T4zkiOP)o$aXL7`>G85|pJ4-mY~Xx*`V$Hq1U}R1Ar*&b?~m-473I zd=}duFDt^?zk-tFktK0#7~@-(Lk}OeW`QLmabm9GKe0DF=L6-jsMx$;_WwMK2*UN> z3(Qq}qtxAN-|!UBh`_EP6PmdVGq3?QsaBbjTEW@<=f1%*6; z!fb|W4$E3^l1mU(?|5G2mfJ~T-x;83Pe$FjTbIcIk}nrP4&x&n&LD6hH%~){rH-e7 zXgMdTswxM87<&(6hoH4~_~ekjRm1#j9XY`>|I9TejV#bpFqhi6`Cs6ec!*^mfA{mO zK^r^zIEVlE;`pz}Kf8bY?)doV=!b9KetmiV`bU2J^ADoRukl|$AN{~y{}@T+`1r58 zIA-o?iRN7BnJi~`a9%)|(|ApQG(am+x1W3Y;nyE1|DXjfn5Is<66E?%iaeV?e+E18 z>r?ZmXY*%&n3Jz$NiuoMYAdel3xx$R2P1Gp7(Ehv_Gh?bYxqSb=pu7f{Jv&_l|0!R zT@Upoqec=TE2`POw_!c&v-+-GVJBugKi&88?GO}VGts{H;*`zuqG|*t87k5l&7@%D z{JWRK#yfBofW5r|F0$E(64n~o0P@|goi)r1SiWG8kK0S1xN0M5UAc|!P{&E3EH zoGI>ss&yIDFzOW+!Gldyr)NO-{lKg^s#kw8Lk0l6*TLrx`XS4LVF0AS|Ay^RhEga>Fs zMkUPC5#Sfx(Y(mm3DHuLOo|nvtDXvUP?J7*9g^>CS zx19qWbsrj~^hWZOR0@odf>Z*11!)bzwIMlIMrt3jJ6>uWK z5$4x_f)#4NK9nuyo-cMJqNC2UaUA0*#;2CY(GpJ$eWT)1ROw5ZWvqmWvWdqr3%~YL zGQggXp))6q55g%Ag55??bkY?YM6pkE^8MPHMoz^BC-IfHQ9R8qf+n|G zq`7{*?N;~I-7c2Z2I3s>Y3O`lOY|GI{#%m@sE>wmykEP!LuudT?%CA7gYOfP?1Qg*wof>JiIg4RewdvZyi#!&}4E z>os)r1oNQlptAVO9jY6EjBKO8tOgHUr0FS%x!UXI8^%SB1Qb>%-MGQQ)}vfRG=W5F z>A^P%crIKgxbP4jT0P4(0=ue`gt6Mx=D16nVMlJiCRSca7s!!ijoHdBsRS7?-3S`q zm82~^?!w61*geF0szieEV?1h0*m3R%>|i^vi0^!fiNsN=Bypu`-NraUfTNC7d0U$4 ztKJAM|5?!F`WC`(5}6lN+W`3!l%;&ZM;spNs8nqCaIr1|0R zx<~IxK@p*`_*|l|6!I@h#t6@ix_W10HIkC4a%gp~ijN7Vt*SPsg$dW5=LCNM0L@pTk?hw6G5LTA5#H(?;n{fiLr&SFvJ;pi2 zPigm3yI;WoXV$dIsgIe}zcEdZ$+rq8w!Rl?Piv319YNaH$E_E`D%^8&bea`wuU${` zjG+u&l*_I8X!ZGJDV3^`4EEdoNpGF>)=6(|KmW>lt0&pqbBMK6)HbqtgT3bw^+Mf2 zj-{g*DM4GHrv*{qceivi7YNtH#mZU1Ke$%EF6kiZk^fkI5ZtdeTF;((I|x0x#|Q%m zmFB4T^`Wx9ZS#AQutG-~)``D;=H!B*iqRe}No7(2ux}s+!+#7PPMM}WQ_Y=s*?r4K zk{rI*#u~v1PN#Rb{aadn+#ww8L##ja!=C-9&iaK&XH1s^@oY&xFi6`>wl})cf z9OMNfYra}LatSSH!Zn75lxeJjj!6?#SE>;oLG}PA{2raO^@{IXO2jDMxC>}Z7U0ge z!=d5rCpJ*L`>W{tmwcWPz3t~-@x9wmo4>rl$Zh=N!m2hd038G7WAEW5*u5&<|$XEkCaV4y$v+!?q7CzWj-#!iYrhWKnsI@=5acyhS z%A?G=c0fC{RfIveib3O9yNoo#%99kkzD4%m^=o0{<4m8me3J9Fg> ztUaUyBPci~MXg}TDK@%E9b{Kgm z7F}qE>KLAH$9JE7IVPCeJ@Mu=e1_19cHa#^;%q-76}PeD4Wb=wKM&FMMz7#lbfjqA zu7#l#Kud}w8o*TQL{Ew&9fz@0qbWrzDbbiqkrc;5A8tR7rBOUA3H+{$ zq?fVJm7U`VvTfWNh?KZEBWcNRSlLSWkflu>9p|~I^Q~#!-Q8-5C4$sO23mFw-%1Q? zD9Yo)wlpcKvXCmS&s6N39x&F}TqBoShuNKR=Kz)g67*b}=V=>G#=1Bf?2H)av+UUc zTcu)ccwBThG5z-N5aT*LFOX3u{K{c*yR|h;NEk-I6~**?t0G&Zs@j`0iC`iNSqZy` z(_o=m_YBXZ8Mn)&t{c7?!jouCYmmuB=XQ`*Y=p3iK_$}*M>EH>TTiKLR@+A)Xb z8D&-Ip3DbGz`LG73YhivxpB&yRHPpp3 zzv3BF8>Jy;Ifi78OcBGow<_Zz0|d0m{ZHgQq=x20Cx2PHZWEXiXW#E)nGpJ3@6Sj{ zA$rtNl5SXOoxy7K{u~LL+xtQ^noet3UU`U%=ij}oD;aL#B@Ke-6GRblfJ)YG&8Xr` zX>v<94xElU5ekjGu2^Re$#&R(liq=`f{x;r0h(q~tQ1fCh}N(hH4@3zR_ZYZhfTk< z;NKzcQ!Jd|>G_%cq6K_u(W6*ORI+9{MIB|3Bh&^`gkz9PuzuU5pbJ(Sk}l#Vk>ZAx z8iGq!g8wvlM`I6aC=f89YjFQURyK%wm<4P^e~Zggi7496RM>IHt+;j}d5O&PN^re- z0USz4Br0j-ePZVFxXql|JW{J5+p3s8NV1NP)zVkediB^f=1jC`N?R9nWth6l= z*Nfx=!auyexEx$mD+~Zrvr&ePDZE)&Gxc07F+_N1o78AbOr%iP(OyH`=oPA3KqQ6> zJs?`n$;*&?)bI&}-+@}0FKN!QmsE9wLhN%{Ap&}4mgQj!8^)1oE$(ZJ>Gjr?rmPJ% zb+O8lW$XrVjRq!;ikj93yU_htR-EI>1F{*uobO{pjJ_?mc$^2WgWiS8f*n;x61rxa z!7APE$!P(aY>o2dpAK3BELr8op+BQjpKSr0+ip< zjjb>0L==TF@qob!w{@daE87A9dWT7m+zYsNd)WrzREYLEp>fcO1~~&`hsowOV6xacppMPHoB>q3%8eD-m3Un0AC2bd7jL3&WL59=0EX zp}8f^&g-m@qU1R(dA0$e3W_!@WobtijKb$B%L}PNd4et5Dr4o>hI$zgw%t=-=3{pk zhg#@M&Pw$ku%a;=EpXr?cH=N~**2hiBV;=StYl3~nrK#XrFnt~!eOP}ZVc3q07clW z?^>4owAgH$=RmC;nAS)kAr?;TF`&dgM;NgvPxwe%LCE_X3mf(rkYT?9T-aEsFdxey zBpJoPgtZei0~CW0VO;=V0h+_`U{T;-ffibZ)JWj3V5fHhtpcE50g5|-V|xPp8VmLX zP+k$+U~fQQBhVhB6Qv&~NqYYY(vOiGJti@FM9T9BJ9u2G^T;&kF)6+y`WlzGJThtd zQTiGe`4y2w8IeF49lvtl*p(4bS&spe^(YWoW8tx)2|o%n)+4}TJq9G!csQ&U*j0df zG!)h@&{at02*6-1|NdH+6p-cDH&nds~u3QkT?;5 zyy}sVLh{xC+*J>lRfwJdPz zMgmrOa(_?{4TDuZXwmwCs&-+hqL^z4q-qQrMm#`OjUO;dN7^4a)xD#FCOvD?vnD+& zwy%+JPmcrjGy>*nG*Wv!#8Z4?PZYdU8@MTgY5>})1GFiE=n$+^CtyA? z3QehdqiQ`Ps6D*wTnmU`Y1>LJFIM$AcinU;N|2kH!KY)zc8MNy4B< zv03cVuoh!rj68Uv2+AY~VYCBH6h-x6@I_;xi$=f}#X}a2n??d=y0LIYqcJYV!W6|q z6t!T7q6gdyPqYg;6istK&_tsrb{r&8G#t@L%q|jwBJwae;RT`)8W{fnxusFkL{~JkWHC=N-i6;R z3xfEp>0bHN=blYzgalY_2c&`gKD09TQF~jf`8~!dc^7WU62@g3UYn z`4#1vxvZfV5k>^Rk4Af5!uO`B-Hsbx8aiG?vohy`W|(PUy7*3}3_E8vkr!KY6xK(# zIszT#f~z&|qIw?dTRa%Gwya8Ewe{D&1WW{DX`PTxgiOs`2Pihf^!Cd3ZZOBf@Oj(= zF#~UptUaHt#U3@ChH87cUC%5g)YdWvXkM07D1dr_*NrwSY~s>(Z!!JMvaQfr;!DEr z3W^dp1%b@mC%KWw>gd9mFPSDZH2wwivcEddesPn`@z$I7+>SpuL+Ig;f!6*9`FPOJ zOFka_0?pyPKT1z6y(%ri?%eAm7C3BiNa)u6Ys))vkj@lwUD=>mR&&mm1WmW8@gb8c^HnH z6?dIbXzjCr4MH&ydL_c%6pw?DSzB#@IRLK>cxF{$#EnH3jJB5ERZw%5mQ;7LHJtPB zUWV*nFjxnk1~z0X;NO72>P3PoB@<4=(QvR{;_+Y4V|tt5r!wmC+&ba_5vkf?gUGggPz&byV=_XB&9>prF%$(y53pX>zSvX7!*h z=crga_jfO?{m>j@4ndcAkSCRMQaSgjayEG_g)GhfgUtFo7+SDkB%|fZ<;Qr&%KFN& z342x*VU81r_JS`{)QEa0YP-$#&q``~V#5i`Fw;U8C9c!%aW$8Wxre|!AA4ZAvXgkW z?4s$*R-)z$BqrQepHUwmWuiy(}h#u;dE!rC55>GYTVr1gKgH7V^a6 zG6>o9s#(*Plo-6cY7`Z_vz#l_jRM1VXl5ij=tPWjT-(4G1Zr&9>YxM~@`%*(X`BR# z(pPAQjTSY{83_;_DQcP~V|DOoQPT;N;nj*3H5*|vKy;+2xsSBR{fL@wb|)e6Hz*`} ze7BXKYx!F2y!2rUN?TPIO}j7 z=jD&JA>vs#!9f#cbWPE<1@T~Y-?jow4$)$Ap{Wst4u|J8%BR_yt zM13Q3RvL|cFv~Zgnu%l5Ju<5Fc_|m{lE>gzYz?*bbq%{={`(eiTo+N9XT(O-*%^un z`(4U1r}_mkH_Xg581c=hh-EV}v=N&w-FtG{Bw8~RTgAJc7o%`mVn$vxVy+P&(VU3E z%5Z^x7W0=grRZw>q}lJ>Os%V&ido5M3iTB1uTk^HMG7GMVzqUW%({4U88cm@hF&tN zhUx(Ij2u71hJfHee71YjT8rjEpI{C8+Ed-& zi3s9#K}b9(P#&RzP~}SGOt-SwPQbPwL6oX3WW?@cUMDd`Fgcuv#;x!FXUBAmDG`f@ z!wr89%`6#36P@>;bu%%St?P)h1+S@T2tug-fjJY@bR2K`MNFz;JJ)JXE+J4rW)`;@ z;%QE5;o-i;fdyl!vZ)AOZHISKvlrwYyS3+pMaIq%G@CUET+0d$zByp3%KZ|VYnEI` zSVRfri=f%J?u+3}OUg69KvcHbZ#iR4%N4t?KtZ;-Pk@@`N-+x-El*haBYC@FVUx02JDtjMfBPHgDD$ zKooK7>PW{bU$7DYB9f98of`xjv`o3qKLngj!&2UhY-5=f7hbVrSBtl6?Vr!T{KNby z9CyKPyKFAb!or3AIv%^CwWvtJ$B zPd`7Bc0PON?V4?xJXecEgtaGnN!FXkfw>?I%l~Ur++@Ds0y}4oQL<^Zm#7175GiyH zx^>@44#5+(X8L_398oC!&jP`90S|d7La+Z>(QIy5Pz`E_f9j=cKW?FLn9pu`mL{~s z5C~gTTuB>NW4KQ*MI-q+*3;|+TgS1WrRGVM(b5%j-Q0T}nr(VvB$2dNU1~GW?XR}) z@`9CIB4X^234ZIK9AR-Ew`b+j@tyimd#Hc|MEwY1bla!L9w)C)?40~<;}X{gLx}xo z7~4mCZ0r&>OkgQX$i_TMB^jzQ&Te?3kLTonu~ORGc11OBVzAtGu5Cqss>oAv1fTWy zGtXIylKHXA3C8^2z6~;bncE2sdd{KCQ9I{5SKXqSF*L$%Z3TOk4)V+n1S~v>9=eoQ zq&mbhYq-gOnEgjd$^L)#{{1;_+)5P3_vic-DCJg3%BowoXOiso-A%rVABi@$N7vHK z?5&;s#%L1Vjff2}1Sz>^v;X_2a1H>1Y!U>!C6ApUwra;|k%xF3ocCh|cMNaOo;NH) zltN_a0|`?x-A>65p`1v0`@LPWPQR6V;|~Av5xgcpuJU)sI}S zY0*de2MKm#5m7tZI0P4!S4y&kW*SBY)##$;AZ!OHs}*vQ$hOK$Vti*MOcZd+FmWmD z5U$l6Jpp6G;6<|xO^wB=+)&=@*^IGgj;Rs)IMqQPqAqAe@W$iR0v%li)K>T$ggdVO zXT0e;s^;h8RQnuL=W~ReEWWnqV^#GWQP0!Vql^H$cO}meTNhis^D*jn##ii&E{+#f ztFx}b838e(N@uJ3X3Rb4t|HWpV>A}K{$Hbc*hDar4b=( zjt!O@WY$3g1jwCG!yZ+8RFpJza*f?h-U{{wh7%84DXoN^NqJl4_DmtLQGbz(L~e}C zZS)w87NP2hw`@B@g%+Q$j}=n~n!1f!cu@4lE`M_AJjA26q8F^q^6*!Q`vzTLjc~P; z#e%0P6HwAnPk`|L^dD%y+d6oCV;Z{6y!`}v-ur=i#UR>Bk|~)0*^0)$f^`&Ns{IBS z(RaXZ2aQ2$PA-V3GRXZ2x4Z=n1MN+4)s3Ncr3pjXJosiRa$wdJ66}uYV^6sE}&1JI}hAoBvr` z5E>H*-BDodd<8KlXi?DD@JCiio3U2|Z9N!(t4PKm7=Q}$mW!A8e51(98GN05HTzKt zMv|;jB`f-m$)XWfch!jV6eR`NVOPu-=xT!y^qa-d3P=@scK+=A>2s}tpihV@cGht+ z`rA+E3WIJ>LvEVwM%cT8fHz!7CzZe4=gFT5ZimY9V7@=Zdvg= z*FG>WSi(|QT@&vyw~=7@v3?51%Q?A(y2rGxRtl`gdD!ZX*G~brpBQ&bzl3Fawe`gw zc|MJ=p6%_^*ti=UDVDnDJr6qVpi-8g^sXU*_b(6k7%lW3VFFz&ZDxnWYK+H2)q8 zfY~UC4Q6F)FoqCaNeoB?6{JSLK)Q>&vVv!4Hp$I+PRN3-DYV8150if{%$5hxRP!3Y zo(JeT=9;SJF+DLC1Rfl6-NHP-Z_`I-&T7a_AVr5#(SyvqgK0lI^L8OLQ}g_K%#j(BL{&hvQBoq(cvSpfk#BZA!n zR}2a7M$u-~ec=TG{lY55S4yf0={wwgP`;$1=|HI)H@F~aYjMQpb&X7p1;F!P3F~pG; z>^Yf{KS?2r=P30GD6j2US?@H0mMOoedy6qEh|34-#PfZjJKIKr~TH5<^{1E5rO*sAb$}1qIv(d!BRE*;Ca6(3ZuzB zDV|$g`g3GN?oIu?H4@!2;J#sdZ_y z0S0P2?`z@2E^$u2uOSddlQo8p;|G4%e{uA&9xbZmmX+3vRBZK!@~R{Yno*JHi_(s= z33MKUk0V5w73L=pYAt2(NiPPK3svf`gcLjKE%YK&@=TK(3wYNUhpnUF-~Cz*C@Y33 zGFIn;InS3l_ZH(1Rhw%Q);VGG6}Z$}sCCrrIkA?i6m-Lm`V~glRgHOg6Rp+Jw8{2% zN2F(3p!7Z%(io21o6Q+%aq{R4zxt!8#^!xBAdEI)`e;yqGePG6cY4cy90v_{^De&T)oEZKb5?LD-Rs0>FxYJ9*4BA6WVp%=_7^N;v@@&&NX4RH z6k1%})2>wNaExn`aVE-}-by-+jd=kNI|mQ@n9IT>sueV0YEG`J+*HsFrr~Kx_rtm@ z^AAK8f{7FOX)&|13oyd(1&5GHN`e_W{(qz@`fCmdc+Z%v( zvt~%yRCD(<%6wS;$n!n+M!&Yt1~8H4l8c^P&t|5lc@zVMj7J_30Wxc@UGhX;$UC0) zsx({}%O3exEd&4V`M>I2=uA`vBkvm~R^|FRdAiR6*NmpDFO6hO#Nwc!+RpPdp;r)# zXp}g(NmSWM4Yv&dhQ4}#^+Mm9Ki%A1*}@e)$Tbje$Aw}^Rm9x=%Oy#<0`=ZlE6QmV>xhy)Gy~={c(Yf~_%SS|f_HHkNoS@k$Mq^WZ;iKzl4T~TDV+Xcr{2HU-oH2A zzm@;HI{p1=_tp*N`emw5NZHw_n>W|Qh$W?jO9<9fWy%j3rrf>!LHd@^$eyAvG@K=< z(u7v!S{C|l0BM8i+_8c$p?EEn>JLS!CpD-*xOQCD4hm{-X+9#s>4kmA22DeGOj=UA zzi};nOht{KF89d97uO-Y51RfW#+T7Ke&?0HF`ahH(S2Cu9V_lQyFb5|g_d|{njOxN zyQy<%YjOU|n64mt+n(nQqa`co7Y0bk=R+jSt;A(|QU6*nnWMZCfh-`=V3{BT>6Z0Yl&HrHCbd{bmONv-vSnDRLaFmXg-Yw#xW482cVK^vT-QtVaoCJgU(^IzG$2XF zV9qVu9=6d~E8?@*=UcYD9^hsJoUChI!$R@k+~fG%Adh<_{S9J zwv=SWL|t#zgMVGpQeU@gR+4{qaZd1B2V_mCA@7D6o|=1va_?))nf{b0$n}QY4!GV;x|P&~Ti)?LDrAgR{C_ zyG4@}1L-mSxO^6UIrvNe%{-Wb&%HKq6(SQ z7446#Y1db4y;PTE#lZQELGuD)W8kS!T_;<Tw@ z5m{W)(_2$+` zo!(vq*8B%koI&vo&uEctLvi@mxDtqFXX1_*5^^iZ9W5{~z~~(bQ{;c0e0TBwFYhki zzB<(ur57f%tF8N>_%ruD(>|bn)NLJ{owq&#piOoin$m5mvZ7hGCF~=lG2ZQaPX~hO zEf?zitEMwu0?&RC^0@pV*#?PHtW;xxcFfXLPM1}gS7k%?1YO89%e9}YThtl3rvu`y zBxJrmEYtWBOxIkEOmmP((=3>*&n|Thcm;Oyu(i>N_&FoHdMCc9aiXM8WtC@EYBIKIL4fnc&!6gaV#-c1^kfI&Z z703>bx>qckdOtKol3?<@9vujJxg-)xOrDu)m{@s$EB8ROIf&yK$7;8BXS}Om{CFn$rv>) zMGc|ZVhPyOU^~s25;`QRj1Gy=F~|;b*f(s$o}w~|6H8en%$T9^k>KtqR+c9Bu4yOv zgan>M8)Xst<;(bevTtDjX#Tf~k;|8sz zo&H3d7Nk5wX6%{U)1cfPuw&C7tAc5{Budb6Pq&2|{X2O{ONufMP@qr_v15o@d|5#A z9C*bOQVx(O5+M7lw=x|eRCq60lKm$=!x#8F*=T6wofIq_Jn){;6xxNJ;Xl7Gc*%m) zD2fGX6d&r`+gOl&!7j%4aY5jWe!%deFtfL;*j7*H zKr;vBo;53W1Sb1puTXTL{4&goEyWE!@a_A{gLri)Xl9pJ>!&+;HT>b46?Z&g0}6w& zQ9T~J-XEFLFBO?=`0FT+UcX}bq3fq_qj3C=a=}F-78rvcxiGW{VCypumCA}F5kP6i zt+7b)^aab;f+_wSMyE!%eB0pm2y@oHsfW8Ge$H@i;T01X)l%s z+tn*JWvu_-S~Y-d(2Jb?k(FA=u^0qg~rXIiA$_Z zU=z+O#fmw8`g7UbAw~z2eb0L8u=H>3d$#P_S@W~=2Zsp`N?^?#KPwJ&ZZ)=>{OR{L zHU}prKjm|d=6#fS-jSSUFr{L@gbICEn)QeJlhl?&2PpD<9y7aQodGtkivSQn?C5yyVc(Q%~t&_ zo8i}Id0OxP=i@kT8s)+UBKBxTIqQiVWttnBECP#h`5t_3tS4%Mj78Y-jR8Q4Fb-pU#xif^JyJinNWjLCaBv+-NDGRutu>m25xaKm4w#2<#fDJZ6;Z-qr z*U<2BPFnA!CIFj2WWUMU=wBARbOa~t4rC>EX~p*ATCzz@rz~LVULU4;U8-`xCY6-7 ziG^$S`sB%zC)kMbx4-=@kp-b7WeMN39}EIU2)};%bWZ-?#oIR!{+O2v8#&`w^#cvS zfp&X({QLDc@9LIT%{M`7blceo91c(OH(!4}Cog#ch1I#~I%6X;ZOE{YC85yX55%Of zZo9@h?BrhsNHFCnc7HMbaq;n&S`~^@nqCVjwK|jAZHL71kgC{CirMa9Zb7>6`ypKRPEc)BGKnl2Z&jWWHVX5z%eazN+@=9M%VYr;~m zMv6_>$_x2dK!LC$Ss3evRaTS{x^2S}&sAXC$!H-Y)UtqJuMdE#R|v6mt)#t-D~+Ad z*K*Gp*OA&9^A+M!C3YZV|GcmP4e2#Zl-ic>UL-gqGnm$lwg^ z3{%gUDTO_4gD^L=xYatMp~!Q{I#I6ZM8Psz@;me{ZRpm|<{F&q@qB)aBt#ZOm9#K- zUjH7F0U`|ct&+RBMlodVw$c-ARl#*;f?6*W@PTm@# zjF*cpl+3CUfyW@j+a^t~r9PHH4>2Ux?lGX^5n$`yJxG*?NWNKKETMa-%0)9+YlLff zP}7ca4G)<-K3{XkgPS^{PCBNnNn_9K3pVLt|B`cv;yim~)XAkuhqO`>?5B|&j= z^1>zD*#M$TQL=x^AE*qEThc7ND_2h&SBXm;Qc<0SWv=lJ5;TDnZe>*% zpO>xvlE^~q++8`*|E`xY z_{yp5Lrrms^EPzY{uL97gIv<8xTB9@Xn)!lMnXJ{J$Gc@zk=Se&bt_ z`|ps1W|?Z&!?0vOETe0Xpu#RJrg4pgc1Iv|1Eq>Nwj*ZaohXWIBqpB`K{moU~s0muXRW1wyeOjsrrbX){_ z+-%Z^riI~?38HR9f&|QSU$Hio_u`%w>BUuV{#a8atDpCUQu{N}ztaV@#m5|mfWvFo zJN8q0!MY4=rVMu5lNCgXPA^x;d`Hv(7u&C4gwz>vaL<@5z!(t|^*eD{GXN|bLqr0aAy1N}C+)~6M zOsM!>bL^JFbEa~QEOp`Ph*bD`)DBp<*0{5eu-rHUH39Kz8GVY;h4#q-4L-=8Zo~ok z!dt7bfjF?Y3S zxDSty=ksF>{HPU%s$9Ht{y0zaDu!m5r#EaPi|tw6uRY1D23+&R zX$-wA7)h$4V4}>nk0^7wof}GKuSfGxYIhl!PaRJOT5QsiV)l@Du1swGE$+x2E!6Oe z$#^j<<#$}kD0B(-p>L0vzVvshCgq5jlhPjn)yIg~gT-5$GkA0fJ2rmh(?zEY0s@!WOCFiR zXrFE-E|hGO%YqhLk{Z+4c0~OPuK>ouMgL%aDK8GmV%)HlSDO)qG~juENA96WttozM zV=C~muoKr5HK)lfYDsQ8v9XGmXM1#H{NNi#hirjpiiSW4nKHfK77)&RghxL<5?us<|H$5G87JtB8sd`iyfK<1wi5pAG;O3xDM8OPU z#!ARwVXkaHM$c^Iokq?d1o>8>qqW~XvwTkIX(KySq53KqOEEttV-9>#33Wymr#3{e zAFx~!p%nzzy#cLA1=AZ0b!-Z`Axh9(ttIrjOK46LUIv=V6*RfkuY%7Grguh?Qxo3o zA6qlUr(74ZTJ_6s)V-Z#RN39_gln{9MbfRH!>PL%SEsl%-XE9;q2$VyUPgb@i;VIO zIZ=#}R~Du5FV|sl8Lg2;GgeHx_E7?Bfq6uE$mebf@arAuoaCuPtIj|7qeB64ShlwS z6RGs#Ixem*m9Nd?oiPQfgNx6kSo!s~>Km%rfgoIHJv^;whKv9w-wX1^^(BJkt93cM zXW-dIo?g=$J4xOSQ0>2E3K&gk?0jgA*Gs&oTd137;vpL$CIgDK!N@KQfMs65g-xgc zVG@ku`e;>m^EDj1Yt)F~6$*DhBTH0tgI)AG8nxkwk>BrX{JCM9JfmfNEXUFJp4Hs* zzUn(oMLMIIM!T!;UJOMFxSQGt#k#R%Rk*H4T2LExM+qVT+bmd`*1c!YlyWavVQ9BUqO%-jAa9*Tw#p8OZfnVhb4IS<{(VHPZsWnf7J`uZ~kufuU@x<-+j;= zr_c{Xv=@dYzPU&IS9cjh=ubb9UQ= zmP`x=EZb~YAxqO#cM0w(y(cGNEyUsnXQY%8=nl}rAj}Iv*hj9Q5(@4u91X=_b0?ta zPijoCg>S-$Dh&$#O33n84)tviAxIpO&^yU(Un@-l1Q_fGw!ob7VoNq$mGqX)$+Z?Z z{LlqFNkO1SKTAm^N*<0r(r>!&!#4Or%H@)%D$z_aFWAhALqo{lMNm!Apj?_LcKxL_ z#mKHa9b7z|4Hf1}wqv0&sCqmh6?N+RDkkhe^@)CA29ZbCvJu~7kdGB$!0LN_Ey)>N!m zoyxtTDd5PTodz&u&9J9Ims-N+B^=8_3nb3t^@DIVP0xnA9osGBSuf)uBm5K_R??J~ zbW~*aTk1MznTQwIFfS4&E@#te0y5{2R^n|KoHOhZ)KV(vOuh|17q4QrIS zuOYEyDM>4k#$(#j=t>}B zH@AG)Fs*6!ij}Gk&P0Bu38Rr-jO7NH9UQ|TRBDVB-HHJoqLPL$Y3@6;_%SXz><9N& zrg8qfR_nI<&$hUfho=4bF-&0E|G|c zLOVu!+G6rm?a(ypI@&GNs{G^LHKg9eZMevwi1t)=Fs@0^bMmW(fMbMVr_p0V&WTm= z=jK3c?WpGCF%E0e`XESV-CC+i9Ex)$TFGBa;su!0ZOWuZ=t5>0D`2~f_8*4^jVdP4 zaRpspXU_cNR29MQqhMP3j4G@qY2y;W>}8`8?Kk5bhxNR$K{CL&5L?{Lmmbof!6ypp zN-s;RJc2GV&CosMbMtNTy|_J<9;EkdX)HDVPfhqyP;1`XfRD_#0etQPS)%|O$=<6M zY0}y1cv#;~6&J!#?##-sgwwf_h=`7{F&z*|3-MraPq(Pi#nmOeZ1utw{140Rqn*Td zESM$-b;TsuX#X3dL-cSo@9>|YZ-;TbxfbmO(_e}P2`imLn?NeICP=Sb^CF$)v?#Y| zqd9Z4j|S|gO-8;nIwoYOm5@Ig=M7tDJ!Zzu>S7b%^rF{UpYmg^GA2jUj>ane)*II1 z@|mtOK4t@|ckn=UWBa7 z=&zj--fyAx2PHy^nMu}n&Ko0l+X!N`DNqq7*%_e?X6u8$xVkh%`b0>9WDlQ>6Wq{} z)~Vbe7$<$$eDYWh>y>4);t?0A<_EwQ!H}V+P$(RlOPRKP2Fx#`CWwK_9C>e$B zgm9rsnq_sbZi4_4Bu?W?@&(a267FT+8w+TjQDdT7RwBVrM_z8n1nKQ43SiYxXr<#z z^d!{Qg*cLG5eY1FTjZi^+j$lgWm8S?wTz2sT+rn9o))Qs0td9@*s<*lr8Zx{6354J zSF}E6F4}UYzvz?tZ*kYOEu)?~*;g^w1FYVJIG|N2H?-slSOEFb^R8`NmkJw85qCvF z6ait$9G1D0-O_3#-nTc>!!A4;Zx7;5$?X%CLNxN2neYvUkboz>V3C(|qm?=eZE%;i zCWLdDdK2C`$T%tCSLA!}p?i?fyqab5UXk)%I;$5-e{sjPG}K6)wbTIvSBsbWD3hT+ z;tnE!JJWalo@JT-Z-`AtJ%c}1Ot6BM;P^BZJo%E)ovAji!rmLdo*nCr8?=XX}Y8eXy-lM&IKz?Ju2%HcyGV^$AS(NA@_$-%e;$cse-MTwa2VWi7d5_*MRqr zm5C70e}|0YQAEt`?3`RP+cU>sbTbGn8n&j`DJv<@RLx6oy7_lB&=I_F%(i;`@Mmw` za*1%`EmL)VsTch02jsg4q8(OUZr95tdCwAg#|lJ%S08gK6qn*P|oUPwpK&2ewc2AJAk8m0*h%?ozNWfiv%CXJygQNmEA(bd;{`DB9?-?@&i*kZPE5v#Wmn@%$Ti-M&x|E|X2+8KGGxi;nw-%b50H#X zc4*HV%xK3S7e+0#nME|*55GO+5^Q)?$-I<#LlZ_{iAL4M^9>br#byrdtWFLJc3<|# zNM{9csrb|V`1#(C_qNPK>xLj_jz3(NSGapa8bxyL_r}QNa>L7#*)V5!Gax=CjjXTd z4?`2cfr4?(?4#?o=_vr9Y2Wie7Djt$QCkqc$*~mz-0hjk!LyEo(L0bpq$Xh>6UI_A z`7e*$jCveP0L0a#XyCJb(NyN63zm_;Rjk+&Ef?CXkk}ZWL<+@Ile=QPbupU2STP0M zbgw>Y*%+!_?WJG0fi#N~!d671oZCiUV#Q$5h723)R@39+-OG4AYD5?#*|8LwbMIP2 zG(>{i=U@P(aABj)T2F4-Hr_Z$OyIR?6>Py`#zCXK*$9xsC4HM?EOO$AYG37#zE)Ov zHqeVEu2GObu*AAK638Ie+a}`|Pxi5U;jm|Rz&5yJZzKWVvh6@c=iPx-t)x(Ep4;}l zaL)}kknd>5$6%4aLR5Ay#Tj`gOZ}f$kU=sMFZFmSnR+Knc>j|e5-$IMBQpLy4t_Pv z@`6emBj-`i#l=#(!{>@z3L*=$yL(&#GfAADbuZN3@R1NZ=4TFE7L5~pu&Fv9_&}=9 z?m#$TKZOVqmlgue{AkHm5rJ0Cv0=pus>3Bw?XT)Fl1Jg#6R*dG-`g?9yoh7C0ao*A zi!|{P@>nHx{K`QPdIO10pGLFY<4kuYH=Hm^9UGc!s`KAkxPS}vKO^*5UZ^>_Ac~7s z#(Wsotml0$8myj?)|38L@jIF^%x9#+^?Z}d9r-h!T40k`EfTogzN1^VJvwXCrD%vp zm*S|d|Jk5HPKJY+h9me}WcgxZGPT+w@Ho--G4-ZHLr{&K_C-d~oig6m--gi9C@X58 zu01%lv&>}9?iit?KptqQhLRg`Nm!!@Lc68N0h`|o%FEb|PGEs>w>2}_QZ1lfB6H-m zG-tNkgxT-ERVN!o!IXtbnBECbhW=l5mzLpveD$cqGzzA2i6Oe3n=nFxSwl-}8@5-H zyee|3Oy8&Wor$BI+*LHT>7kgl>@EnTkeamRD2Nx$8g0hG1cj%JZ&`M4L=ve-u#l-UCV~o2oHuYFc)v8qK$AFcxxe!Q)AMR{zhho@-+v}I^c*(e8GX8*iTnkwV zgZH+|*Wo*=S8r(LC$y5xmnt|uPT(Y9Tvs-MtDOc2IQpxb7;Y(+D zcHJ?YW3uB+gf1ml46m&!Y01v8AL25zZBwF|&_E)U(Tv@3l>6goAd>QtIO$M_B|AF$dO>x2TL7(5YBQpMj2*mJUil8_ zsa%rRacv|$h&VAf=oNEsV8p#_us#ZZ?<+^zteY`_p->+d!7y#c~j&>13C?@B*L^isuOJ3 zczJ)%(-E5ik2g2Ls=j&?h~5RlQzlA%xr?xzb^_2&$t#W}`2aSwXW1;csOvk{2QcIX z()Y?5`V}j&Xc!cIkmZ^vzNxa33MMO+ZG-6aDGymF}1TS(jDi9N7E zwZR6c)ET$wx#PU9Xde5>5>sp?&#D#gEsga_`{CEEy(u~t zZaUl3la%F54Ak?Ig1oNr6gTAHy*GWle)?9;$F`$bgJXCa?pkK4BJ5*X(BQ`tx!Gtz z*1=m6D#D~IN_BWBVIcj2g#m1FKaio^@-Wy}n!C{WIZng03GLWszuA``ZYC3oL+`NI zwyLFsPTsB*ClF(EvsAOt1sW*%l8A8NVsmzlY+h@h>s&zi>Tj@yI~9D-E^@h`jSLIdx6Uc;_)6n@=^`$|mnvamU&HxyksN-ScuiGs(*5AV8k~ z68q)?aTh*z`}ZXj?2T(1 zWS(tVG|Z1i$nq=|{nm^g3x>I~hMU?_cJ>3txE`Qx#T~baoTR9P=s&b5(TtWo)h5%IT;0aC z;o5nvdA0ud;uSH!MrtnIcETY;VLw<|SByWfr_Cy1wJRm%>h_SFky;CRg|+6@bE}^O0gp6cw#}45*bj{j z!H-4ROWQWlAuZ!^?4>y^N>I1rk<-0mNul?HI21cCmo}lN=?CY^Jrq{5T@5+Sg?tQr z?fSBi^OW74tCT)HgDJ#h>o(U(6*+o(G$+@5!!ufBSP;xrMYGR^P2#d}6M>+%Cr9K& z7BHiP3X(BfSjX)IUhYp_M5u2}NR(R2dEVjnMi#Cof-Jy?^!M;^x)M=j4y(3)~l;=j2A# z=Hh_D<5K=iLlHN(GbF--+%aEXRiA8q@J#U1;D*U8J&Ukb}iZXZjT%S^7idoYjKh+kqb@{G>( z4j-=8lGyA*n50XJmk*)&B-8@_Sbdw0?1)I6S zUKK6BHkYYa*EHn6)&HxmbI}VJ?#H`tZlG~5_=^u#bo4|5C_%_!J7fAo+R^3T>_J zPAc>;avswv$lfFTO~1&V@zW6j)IkNg z&o9!)Okp(-BdaEBnqO4q`XyITgfU(~*?H5;!N5+z#v4Jn-sm8r)-YcD>EepeO7C82 z+B6<08PsdM#vJF-D>UyNus#dnF;@4m;qwhY)_~=mmV5ZI!Z6LuM*FeC{;N+Aihm%d zfgAmg;=AI4mz1Zy1;{`C#3OcVNuBTlg&KKWN;R)lUJ+3%i5w=b_-721xk##WR_Ay_ihG&k-@{uWR6_}@g! zQi@CczZK1J``$f-nXkG1@w02ZW+z%7!#ig$snkHU7Zqk@WkH3~=hpO>92+O+F*zxd zJm}wh@En+|627>C^Sr_1c;j+lLQ{HRCm>*TyhY1A@6N}c{}ud=yx|H$bB6lS!^2#e zwy+?%U1_D0=YOGTT0BP{yfl(jF_HHOz}T52viAm0E%NNcVUFQ-khv0r#> ztW9r2!W4~o{!5VVy-VaU(*yq}jcz-#{rzB;XmC?H^}dJE z`G#qR!A9_{ubVMSXe%S8$oN8QI0ON9XA2<;%SPS&S*t(R`&9 zL1X^bP9p8sMn12*cMW&1)o-uw4u&OSr?if|f?+?1^SiaK;OBMq_UDoVRT&)H46bIc zstgC~ABgfJXpVis7&7$1)f9(<(aO6^k4;7FZ1$!kh5Rd&3XLNo_GvWUW0R^Z)2AYZ z-v-MRR_G76#}-W+nqwPylFEB=Pm46rEIKO90TFSiEM_y&p?(K~>%gb0|Hp!truMU` z5bf#OVONQ$osyc2Xp&*R;@QOnkx5B2+uD+q_l$+5$tO>qpzZO=Z-4t+0u91amhep* z2*Ia*{q*Ua{J)F0p^$!J+T|*&ScNl!1MvZ?GxG1(-@L2qq?*450h>zn8b{oA&dEzJ z{LZ)a(Q-Z5VF}75FPC0vTdpi+umwl>YHb^kYaGk?KqYxFr=VTf*1W+mOZblCB(Q`t zlJQ$cmI_*rhjU464dsF1-+M^dc5HPmaq-NN5)=~9`^%;GbLuK5uhWDXvX}kNe2&l$%uJ02SgjS`TK^b}* zY&|DG2m*DL&!FycOd)e4LG8}RJAJ>Fa!jq;A4CT{H+y~kjH75_yK2QwRnQGP)6~`$ zIBr%8RD2~^f%cfVrct!jvkkWJ&nQD{9;0u)B6PKa?jm4K-^7u=VcmNF#io9OH z3Amo&f|ms?wkW*Vv(v;1`x8X&Fw6?OHA41PSllP08`=S$U61H5*`iPQDu{TymrIRl zY6lH-kYTrxpP+w)D#C#0MndHF>bW>IR?f8sS5`F3wlnrolV^U%Vwi;;n9^$cA+Yic zkvcC}iFJF9a2-NIM#d358VMR27gQxyJ?I0>?pU!ku%qCPZ*aJ8|IINSIHN(5K>Ny$ z9-JfKj&4=7M~}0|=)@V>i*ppjTV@DVs2k^4hyZYGKhAwSa_-TS^O3r8K2TrI1DfOQ z*PHXvx^q5Kf6m9~(0NFY&I1l?pFW+B*QxWddUZZVx6ViG*ZC+NJ0GQI=K)*x8%9X%hTr)PXu&oLlP z|IY{L>$%6h82!Rw-96*t#&LA&ycP-fvYHVz9hoZCMyu>mO+7U>60*jMhH4*oY$LC` zwB^Rf@1f?}Ma9lYD#dYW%saVEvG`M!$E)~W2)|~gyN7Y2PBr?9<5=oU?Gy+meQDNeq@F0 zGWv``wISbYJ%+-G>(Rio_n>*)8TS#iV)CkX6;ejG-Iib=Md#lFR{AJ+sy?jG#Tw_C zujdbAJi9T4Qn1R&(iGY^4JkV&9tqv1dsd=uQrfWdKKyyb3#GvT-S*d`cpy5+-_m@f zOUb?G_i-R}A*<46$Ku~3whJntm(qIQ9@scNLN#fovO?%VGxp_-Fq*8j2sQ&DClaT$ zz-cARmwK_zSJvh`BX*5=$m#`57{7~)u~H_;C>A2fn8J$ywl+9(*X+-*{#ZF~w?lQt zm7eY`+p67F#PSon);e`uUk?-NQ3>xGWA+pJYC#IWVy3t5!_KQ{g^r@xG8kTdr?~B{9<)P*Fe;vZrSx54!nSC=EL3ErDM_5|;VFV>=Pi53MffQh`Um-XEk@h4iN-(Pv! znc8d3D(g6oXtv=WXemv~Ld|^Z=AiLTA(KB0gysh(Ye#oaBcml)%D3C*z7&xm@5Qi->zIA03=zt|AI$v%VW>o&%Xp2{{7=wr38Y{^Ll-R?o&jgJ>g? zR7Jr=nQcF@WbXDaK7^zm%>#9v4bej8WU$fluL(vUjaf5Y9*Gp8?& z2C4O{r^9)T&5_fxWQBN6{{JUG{PW*ur@ucr`Ohb_|NUS8eDcE_{_(5R-=F^7{*Qk? zJv}-3&p*HYHmoJ{K3t2;f-uWu%Q6 zU+OV0n?I(sj7fR?>@3SaMXVZ@aF2vK1*(0)fogct4buR`pN*MoM3frFpG5+VjZy;_ zcQooxQEJ0uic*`R)TSu4F)*hnwJAz%ic*`R)MCaIr8Y&WJwTKiDs3IHYLSaG{kiHu zwSS~IHKh?OVHZgvD^cFaTNV+3(y}G&K!6rgZQDYOEdi_#yWk!-gpF1y*J}J8IA&k< zn>&h>rS1=rC}yZhsaHEE7o=bb&$%g+K=F6PI#c*KmfrGGv22M_5fydWS zks{s4aJ2)Y7MZ{u7HwiltHPzK-_eYxyxh8oFuYz`_ZY0qUJr|ysy=f<^SqEWSu^F$ z{A|Q&n-cb}xNTOgxIm=QS%Gyl)Qs**f?LN;7fWo(H8Pi6Vd*CVcByt1-qQ3GE=fqy$KCAxAe0fkxPTP`^s zrLV0R)-@hgL~NP6FC@)A^1)$Ieh|>#D!JH}ajiPcv(3_c|JVGu(>}0w-d}27 z9@z6zjI6RMWECt*&{zvRZ8c=;QGklp`bXOk3C!EUlL9z{NAze7F1mVC*b|9q`Y-qB z;1`7WKy8Byw&JQR#_R~b&_qsCDoGY(Q)MO3Gq*dzMZCAb443r1Bh)j!Q|v5*EnRcU z!^HQ*&4Gy$`V2q>J$;pjo{OUhK=E$XPN$MbLNJ z?4k>g6L~MNRU0EKe#b-$D@laO=npixWg;ctD!V!9R?v-Ue47{Yj%(y(X$$s1q%4}I zu`aVMosMGFp*vNn5e*~t!$b5(q3OTicL|G;MK#Pr7jYMC-L$g|dz{{_Fa>;#-xn)` zW}2*o^)cI-q=hJ^(EXz~G`3f#_kkce9ptmcBCeCoaj{67H0(W1r6PZTQe`h02<<+v z?cJAl3*X&Vy`2q4*+;s`Gp6RZ{{pr)bEnRWMSA}J)y2!VujZTdkS3EOn`j+}JJ>av zJBN9l-iocP$UVh6i>CA44bPvGS3*_LVuzNk&kV|GKuY5J;E`8mbHftlC+LQEXlvPG z5!SQOgu0gZB$ebILJsbPC*RQ32y}$U zozVgACjMk9mBm@ON8PqxuI2r#l(R~)84t$3MJ@Q2Z6OdaddcR6hVEBNF!I9hTG)91 zhnL@~A-j<$R-dynIZs%=J~zUMu(oZ_vBj zU$6iAC)=jsKI=)r%7GM+cB^&s3h3vaqmfqSnu(GpXxUZ60f#>6B9}|lDpHn0@0)_B zOc6()Y9tJT6g>MByBq7iiE-eLbKQPwr|ejFY?PyQ6rCG1#O>!?9HlQFgTmM#knqb6 zfu!k%59CLG79?_5B?Gyt_pg?z?yHfhgl6pWn<%Z!Jlj;vuFa2Lf93%-q_0Q#pB?fL zA=7?G!PFnTsS$Ztk8O(&W!fuRvU?iOv(58%`(Sl6O$(-!rJx2&f=j%(x+H%@P>1&^ zFJy`0=S#VvJQ%3Z?xEXw(JsnourZ6da&U+Nw3LNc;idA9D>>Ih_wh@>{AKX~V zgY{@9Ip~X*gOti%3A)I_p4KKpyI08TjIIcS|8bsjrTG7hFzN8PE*Exh4qgo6)ljy7hpXtFmw;y%<>#BRKZ{-7)O`CqmwdVdW>75Zh&LX*t&MH{)Ic+LFt>%0fYC-ZmfXf57h_ z;)e2aY~porKQX5}yDO^lRYx|L#cI%?w6f-qwWUG*hs1p&7TTRlXvs;J^r zCaEem@Z`yFzW#bnUh)DO3~|>%yLMU7f`KrKSx%sn4wD5V#0RQUb!1Zw`ypFZs1D7Z z+A~`!HajXVz;MvJ(2;?D8O%Vpx#a`<8I^ZFG8ely+k7~WJPg;*F@MnP@ew)Wel~az zW=y8|ZVve1arNbg4`hCa>0dJ|{9(G1?AwbZ!qGM|qdPKFJ3gr+Dl7Y;aJf@gtFeiC8&o}~NM<()Mc5wt)X9yUTQyfj`;&KayvxXxw4<0O; zATHB%WPWL#IU1I3#1ATg^9Hx}U9}ZG5EtPSnYF`VhX;hm0ou>AZO9&(n*i8nrd8F%F0ah-q>W z(;)+>ue*O2SC=Fy7-SC9Okq>#Ry`ZD-9e~ETC_49Elak^2m0VnPP{MPiT5B5fpxW^ zVy15}xbZx$8r*?46PXipSCO@XK894KgCLH1@1p;@g86*Xw#t_G*+Gje_IdVMCHe8I?u7lka0 zH^*N_5GF*PXg!e?s2UX}KH-{`21Nkr*brf4PNg->&7jZ!4mfvQwp{)rON@6a*Sz+-V2WAu z3ixj{CC9sK_|v{}qp5O$8Jsqmpu9v{AC<#JHgW^~PHN;0`3C*NPZIqjG#vd3#d^L#reKM3;u z#hV5^u1p|U>Uq#X2)HWbhLvkpb+OIg{Tq2UfA(blM8kY5)4C-t{-RVN! zvH32U3n_i~Y%#DyFTt9n8ey`oJP`<5^=$6hg7j5s3Uey0>)=szBVp&1{&aJ5MRHou z4J%oJZh+7fhNl5jVKil-B)i63-k9!}<}JL)#<)(?6j=~Vv|uWiLWPR$#3`G`nS0HW zs^H}oiQx#B9>{^%>Q%#;SwJ&GR&goAo~hQV$hShVQZv#;M#XKIjYsN*`hldpDX36d za9IS9*j_HV${1CQZ0N0)bOva*m-JmJhTc4^vY{nzsGY%; zJqh$Dfdyy)=%SGK3eZrJOw!c9oYqBz+C9y+utbN0LNHw)B~#$Rnjd4_A%IlX0{@o+ z1q8O~ddMANO&ck@W0}VJ`9}W8vy9GVu{vjB_U$zqI=*L%a}DwQPk6u1d(ZJ9(CwqS z5?TO8kL?&&xka`AnAw}9;mi!6hq_ke0$i#rt;>Z7sRUmxA8*sUR}}ml6IC#vRP+6# zut+{5@8q>!*n=GHYt1GOIaC0!dz#V29Y^>y(IAvEfnhuPuE~P0hO*`kJKM&>9{*5Z zGBM;hyw*nr>wb`<|BvhII`wC-v;U9l>zS85R3ARgvi;BgkV9+E=jW7;TtW9Zr5`zz4uu z`;y7aND(NzGp=8xt&MiU$(o8)I9P>}LS~s(1*$xyWo>jU3%**hB4k(?cFhbN(S~aF zB@iKcNL;UEzo8%brrHouZ5FK13s-zoWhE6%R$4W3fO{5jL4IU~H2bz;)E4gd1dvoR zC5}rI69i^WE)u!vN6Y%N-TO8ihqB>9uj@H^64=uI(3G66*bOhU9@|A@J8v*G!`FrM z>!j50ch7&|W|o;6trO|Pym@ooR*M2s*|5@)up&@ptuYPLBC(!_ewQyHQOIRm4i)l^ zR)dy&^_B)JEACDY-|<@YyYkf5MLAqBw4}v~g*~m08}N8jM>=BLxd#n&jH|5wd3!S- zrB;4io%|6y9bX^6qk&~JSgA&BtaKl-oPBEH$JfDss5r9d(zN^5(T{1t*OCh%sv%55 zJ|7~{4E}N?0}7kXNo*f#hy;Rim=R&54oALCm_S%5^xXvUwe@L9!w_23UCFT2m04L%MZH_LSs> zn*fHvai-?vH6*}->#(FegR&jA7-zgGi!Z>tf9z#i({7K86~j6 zVQh$DlCd47hL~fRF-9@PeHh|~el&*{NeJE=FS2K3s>_~%*H;tOx^hyO)7`wee)`pr z!#s#48!kpJ0Nuq8xPWH|EdW^)N)Io_TLfQ7Q5G_5RcJ(61Sz12>jl<+99qsss?toS zXc-pINQ1&3tJcFITagC_OS2Xgt8=&61Ci(FEY6}-%t%$v#YY;i>j1`!YHY8 z#aA<0Bx_!>q^t^dPIEp>q`1>kv=nN-Nq?zwmdvO~XE^gFx9Jx%sFnKOGl-4txMfY@ zLY6f1B4*tCB$uhebZ-=YT*uW77kaO@rbBR7Dgl}&Y>pIfMEj>K&*WC4@h-B4V_`yp zE_y1wM?=_=rjn}A0tX-k>%fW76no{@HqpZA_(^q`{o{SIU|0`R4V6!I8BX;i&&Wl_ zigE+VK?RnXr9!IrRmKVjxr-hQ$*37s7_ANqxZ#nxguAcV|62RN-XvJBNL|RchsW3*?Fyv#q$j-X-Z2vU%6-ckJEO2b4w)m)bxj>THmd5DNz0A z;Mvj)k}t`JuCILPS1gsqm~2a)K070e3P1Z{jucx6sv@!AJv{1n7kF^|?BeA;v$<~YZ#S8} zKV1e4Urm1DwLz>;f5G(KE*QF~_nbvIp>UU;c0PuC*!d^@`7Y+XUrJ(2{mQs1K?l2K z+m0jb`T8yh{NRSRyFBMq%d-Q_{C!Q9tad=fmQalHwb?$ENLi6V^L%8NT};-P4Zuo< zr<=1vUpkh~i7AGYv64C9u6(F_tY`AkSLD_ODO0#ctrNNY=59jogUZpcZw zX1pL7U9e1%4J{K>lMU1=S)4wkgl2(8$Y=IZTV2wd+8zt;mxTiNQ>|%{BGKuRlXj-8 zaha^C#yM<24Gq2#%$&TU$=d#ky*j$)Oo^&C8(P$+KK-`2;Ugr`AptK!N9bDRXicG) z@wGLGU3~%fZI~7IR&I=8g$FFY6AeGbtl<;=wHO+czOSL?|?`>76Zvqd4?+%31Q5%MVK=JE!0IR1x~4LEp~5b)Z-G zbIlX_=_H)K#l#Ca0AYx|T*E_Fncmdat!!J{x4fJuc>EzbchtQDOQX`sqJCdG_S1ub%xAdGY|?9vqb_Y4J}_p3q#%;*Rz{ zr%Jk92H-rw;r{!Dj9)sJPC@VI=+>zeD@=JB@bJ_)F~c%%vgrKn?#cYO&*o2Tuivg` zW=WBro6fpitmcUG8+39Gjnvaw6|%~olkSHQo}D%=*$Uba;s2hIj?IR@;O4ZX z3oVoVx%G!P+$gMhRuwdBP8^xVS{6{1yzx=|6Bny0qlN$0F9_BUeO?!p)ZEkNt!NlC zLQ^y-((J0x%({3XvuY#U7k2CnRwj_kU6JXpN(!Itm}kvj@i@L~{I;X=b$qcFd8}O0 zV*2gRw*HLYeTnZjzUxaT!4IQpA@Et`oqaL~&fvY*`mXEyyYM`J!S7H0e{(4R_;3G4 zj^DFHiiBq@JwE-fdBXr}KV={2-{<(xxj??2uerZRea!h(peU)aEUS`k@+Qz#6P$C{ zwGQfgFDS!`Qd&KM^>b=nB`7=+S$IuNc6Xp5V&Xim{GalIODLU1o;D`uTUhb6JA6PT zYqoL8Nit{R;_C9dudW+Ew*>u$gpTzH=#X=`+v4UC?}mKa3|G)IGM75_>K~ZPtCfD3 zEt+t+s$FqiUN^af4muzN&k4rdDBD61A`A-99_32#A02GhtT>}3+p*gLT@toY&6(@; zR51M&QV94WKOUX2zg* z4-x32A{H~XpzeP=D?A50H@3Qs0?%?*nRpl;#Pp&Iti#`gykfZL-gMFd%^!bZE)*Nf z6stJZVK2|EtD?U?7!SR$`2+Uz*0Nanzo`q7L={BNPUw6&S{y`^pTS<-0~KAczd6+- zX3vwgmmO#Hxc0(y0{O6KoJgfPBy|*VNyqV(xXvtEJQu%2qZ&SGohvanmM2jWGl?6; z8-zd7U>{^4wrSl5v8HZ8rT_4=6