From aaa3a8ce4baf5ba51c11c6b9fbb71d5220282274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Mu=C3=B1oz?= Date: Thu, 9 Jan 2025 09:13:43 +0100 Subject: [PATCH 1/4] feat: adding cronjob to delete Airflow logs --- charts/deps/templates/_helpers.tpl | 3 ++ .../templates/cron-airflow-logs-cleanup.yaml | 48 +++++++++++++++++++ charts/deps/values.yaml | 4 ++ 3 files changed, 55 insertions(+) create mode 100644 charts/deps/templates/_helpers.tpl create mode 100644 charts/deps/templates/cron-airflow-logs-cleanup.yaml diff --git a/charts/deps/templates/_helpers.tpl b/charts/deps/templates/_helpers.tpl new file mode 100644 index 00000000..975bc387 --- /dev/null +++ b/charts/deps/templates/_helpers.tpl @@ -0,0 +1,3 @@ +{{- define "deps.fullname" -}} +{{- printf "%s" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/deps/templates/cron-airflow-logs-cleanup.yaml b/charts/deps/templates/cron-airflow-logs-cleanup.yaml new file mode 100644 index 00000000..9bf92143 --- /dev/null +++ b/charts/deps/templates/cron-airflow-logs-cleanup.yaml @@ -0,0 +1,48 @@ +{{- $airflow_cleanup_enabled := and + (not .Values.airflow.scheduler.logCleanup.enabled) + (not (and + (hasKey .Values.airflow "workers") + (hasKey .Values.airflow.workers "logCleanup") + (.Values.airflow.workers.logCleanup.enabled | default false) + )) +}} +{{- $airflow_persistence_enabled := and + .Values.airflow.enabled + .Values.airflow.logs.persistence.enabled +}} +{{- if and + (not $airflow_cleanup_enabled) + $airflow_persistence_enabled + .Values.airflow.logs.persistence.cleanup.enabled +}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "airflow.fullname" . }}-airflow-logs-cleanup + labels: + app: {{ include "airflow.labels.app" . }} + component: logs-cleanup + chart: {{ include "airflow.labels.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + schedule: "{{ .Values.airflow.logs.persistence.cleanup.schedule }}" + jobTemplate: + spec: + template: + spec: + containers: + - name: logs-cleanup + image: busybox:latest + command: + - sh + - -c + - | + find /var/logs -type f -mtime +{{ .Values.airflow.logs.persistence.cleanup.retainDays }} -exec rm -f {} \; + volumeMounts: + - name: logs-data + mountPath: /var/logs + restartPolicy: OnFailure + volumes: + {{- include "airflow.volumes" (dict "Release" .Release "Values" .Values.airflow "extraPipPackages" (list) "extraVolumes" (list) "extraVolumeMounts" (list)) | nindent 12 }} +{{- end }} diff --git a/charts/deps/values.yaml b/charts/deps/values.yaml index 3e7a944c..4439f674 100644 --- a/charts/deps/values.yaml +++ b/charts/deps/values.yaml @@ -139,3 +139,7 @@ airflow: storageClass: "" accessMode: ReadWriteMany size: 1Gi + cleanup: + enabled: false + schedule: "0 4 * * *" + retainDays: 180 From 69e34db17abd0f806792c4220c24888452983380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Mu=C3=B1oz?= Date: Thu, 9 Jan 2025 09:19:39 +0100 Subject: [PATCH 2/4] feat: removing extra spaces --- .../deps/templates/cron-airflow-logs-cleanup.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/charts/deps/templates/cron-airflow-logs-cleanup.yaml b/charts/deps/templates/cron-airflow-logs-cleanup.yaml index 9bf92143..fe21648e 100644 --- a/charts/deps/templates/cron-airflow-logs-cleanup.yaml +++ b/charts/deps/templates/cron-airflow-logs-cleanup.yaml @@ -1,16 +1,16 @@ -{{- $airflow_cleanup_enabled := and - (not .Values.airflow.scheduler.logCleanup.enabled) - (not (and - (hasKey .Values.airflow "workers") +{{- $airflow_cleanup_enabled := and + (not .Values.airflow.scheduler.logCleanup.enabled) + (not (and + (hasKey .Values.airflow "workers") (hasKey .Values.airflow.workers "logCleanup") (.Values.airflow.workers.logCleanup.enabled | default false) )) }} -{{- $airflow_persistence_enabled := and - .Values.airflow.enabled +{{- $airflow_persistence_enabled := and + .Values.airflow.enabled .Values.airflow.logs.persistence.enabled }} -{{- if and +{{- if and (not $airflow_cleanup_enabled) $airflow_persistence_enabled .Values.airflow.logs.persistence.cleanup.enabled From 108867719b9499cc760001ff6752c26de04ba510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Mu=C3=B1oz?= Date: Thu, 9 Jan 2025 12:30:38 +0100 Subject: [PATCH 3/4] feat: removing extra line from yaml --- charts/deps/templates/_helpers.tpl | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 charts/deps/templates/_helpers.tpl diff --git a/charts/deps/templates/_helpers.tpl b/charts/deps/templates/_helpers.tpl deleted file mode 100644 index 975bc387..00000000 --- a/charts/deps/templates/_helpers.tpl +++ /dev/null @@ -1,3 +0,0 @@ -{{- define "deps.fullname" -}} -{{- printf "%s" .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- end -}} From 77d67e91714a5c974df3c17ef04d7be9198e08a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Mu=C3=B1oz?= Date: Thu, 9 Jan 2025 16:05:25 +0100 Subject: [PATCH 4/4] feat: removing extra line from yaml --- charts/deps/templates/cron-airflow-logs-cleanup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/deps/templates/cron-airflow-logs-cleanup.yaml b/charts/deps/templates/cron-airflow-logs-cleanup.yaml index fe21648e..76cf0002 100644 --- a/charts/deps/templates/cron-airflow-logs-cleanup.yaml +++ b/charts/deps/templates/cron-airflow-logs-cleanup.yaml @@ -44,5 +44,5 @@ spec: mountPath: /var/logs restartPolicy: OnFailure volumes: - {{- include "airflow.volumes" (dict "Release" .Release "Values" .Values.airflow "extraPipPackages" (list) "extraVolumes" (list) "extraVolumeMounts" (list)) | nindent 12 }} + {{- include "airflow.volumes" (dict "Release" .Release "Values" .Values.airflow "extraPipPackages" (list) "extraVolumes" (list) "extraVolumeMounts" (list)) | trim | nindent 12 }} {{- end }}