Merge pull request #57 from godatadriven/upgrade-airflow-base-image

Use official docker image as base

Author: krisgeus

docker/airflow-python/Dockerfile

@@ -1,59 +1,16 @@
-ARG PYTHON_VERSION=3.6
-FROM python:${PYTHON_VERSION}-slim as airflow-base
+ARG AIRFLOW_VERSION=1.10.12
+ARG PYTHON_VERSION=3.7
+FROM apache/airflow:${AIRFLOW_VERSION}-python${PYTHON_VERSION}
 
-ENV AIRFLOW_BUILD_DEPS="freetds-dev python-dev libkrb5-dev libssl-dev libffi-dev libpq-dev git"
-ENV AIRFLOW_APT_DEPS="libsasl2-dev sasl2-bin libsasl2-2 libsasl2-dev libsasl2-modules freetds-bin build-essential default-libmysqlclient-dev apt-utils curl rsync netcat locales"
+USER root
 
-ENV AIRFLOW_VERSION=1.10.5
-ENV AIRFLOW_HOME /usr/local/airflow
-ENV AIRFLOW_GPL_UNIDECODE=yes
-ENV SLUGIFY_USES_TEXT_UNIDECODE=yes
+ARG AIRFLOW_VERSION=1.10.12
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
+#ENV AIRFLOW_HOME /usr/local/airflow
 ENV WHIRL_SETUP_FOLDER=/etc/airflow/whirl.setup.d
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-RUN mkdir -p /usr/share/man/man1 \
-  && update-ca-certificates -f \
-  && apt-get update \
-  && apt-get install -y --no-install-recommends --reinstall build-essential \
-  && apt-get install -y --no-install-recommends --allow-unauthenticated \
-     software-properties-common \
-     wget \
-     dnsutils \
-     vim \
-     git \
-     default-libmysqlclient-dev \
-     gcc \
-     ${AIRFLOW_BUILD_DEPS} \
-     ${AIRFLOW_APT_DEPS} \
-     nginx \
-     gosu \
-     sudo \
-  && apt-get clean \
-  && apt-get autoremove -yqq --purge \
-  && rm -rf /var/cache/apk/* /var/lib/apt/lists/* \
-  && (find /usr/share/doc -type f -and -not -name copyright -print0 | xargs -0 -r rm)
-
-FROM airflow-base as main_image
-
-# Optimizing installation of Cassandra driver
-# Speeds up building the image - cassandra driver without CYTHON saves around 10 minutes
-ARG CASS_DRIVER_NO_CYTHON="1"
-# Build cassandra driver on multiple CPUs
-ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
-
-ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
-ENV CASS_DRIVER_NO_CYTHON=${CASS_DRIVER_NO_CYTHON}
-
-# By default PIP install run without cache to make image smaller
-ARG PIP_NO_CACHE_DIR="true"
-ENV PIP_NO_CACHE_DIR=${PIP_NO_CACHE_DIR}
-
-RUN pip install --upgrade pip \
-    && pip install apache-airflow[all_dbs,atlas,async,cassandra,celery,cgroups,cloudant,crypto,dask,databricks,datadog,doc,docker,druid,elasticsearch,gcp_api,github_enterprise,google_auth,hdfs,hive,jdbc,jira,kerberos,ldap,mongo,mssql,mysql,oracle,password,pinot,postgres,qds,rabbitmq,redis,salesforce,samba,sendgrid,segment,slack,snowflake,ssh,statsd,vertica,webhdfs,winrm]=="${AIRFLOW_VERSION}" \
-    'pymssql<3.0' \
-    && mkdir -p "${AIRFLOW_HOME}/dags"
-
 RUN mkdir -p "${WHIRL_SETUP_FOLDER}/env.d"
 RUN mkdir -p "${WHIRL_SETUP_FOLDER}/dag.d"
 
@@ -75,18 +32,19 @@ RUN mkdir -p /etc/nginx/ssl \
 COPY nginx-ssl.conf /etc/nginx/conf.d/
 
 # Harden Image
-COPY harden.sh .
-RUN chmod +x harden.sh && \
-    sh harden.sh airflow
+#COPY harden.sh .
+#RUN chmod +x harden.sh && \
+#    sh harden.sh airflow
 
-RUN chown -R airflow.airflow ${AIRFLOW_HOME}
+RUN echo "airflow ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
 USER airflow
 
 EXPOSE 5000
 
-COPY --chown=airflow:airflow entrypoint.sh delete_all_airflow_connections.py /
+COPY --chown=airflow:airflow entrypoint.sh /entrypoint-whirl.sh
 COPY includes /etc/airflow/functions
 COPY pip.conf /home/airflow/.config/pip/pip.conf
 
 ENV PATH="${PATH}:/home/airflow/.local/bin"
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/entrypoint-whirl.sh"]
+CMD ["airflow", "--help"]

docker/airflow-python/entrypoint.sh

@@ -1,18 +1,26 @@
 #!/usr/bin/env bash
-echo "========================================="
-echo "== Initialize Airflow ==================="
-echo "========================================="
-airflow upgradedb
-echo "========================================="
-echo "== Reset Airflow ========================"
-echo "========================================="
-rm -rf ${AIRFLOW_HOME}/*.pid
-rm -rf ${AIRFLOW_HOME}/*.err
-rm -rf ${AIRFLOW_HOME}/*.log
-rm -rf ${AIRFLOW_HOME}/logs/*
-echo "y" | airflow resetdb
-echo "Removing airflows default connections"
-python /delete_all_airflow_connections.py
+# Might be empty
+AIRFLOW_COMMAND="${1}"
+if [[ ${AIRFLOW_COMMAND} == "scheduler" || ${AIRFLOW_COMMAND} == "webserver" ]]; then
+  echo  "wait a while for the other systems to be started"
+  sleep 15
+fi
+
+if [[ ${AIRFLOW_COMMAND} == "scheduler" || ${AIRFLOW_COMMAND} == "singlemachine" ]]; then
+  echo "========================================="
+  echo "== Reset Airflow ========================"
+  echo "========================================="
+  rm -rf ${AIRFLOW_HOME}/*.pid
+  rm -rf ${AIRFLOW_HOME}/*.err
+  rm -rf ${AIRFLOW_HOME}/*.log
+  rm -rf ${AIRFLOW_HOME}/logs/*
+  echo "y" | airflow resetdb
+else
+  if [[ ${AIRFLOW_COMMAND} == "webserver" ]]; then
+    echo "wait a bit more to let the scheduler do the database reset."
+    sleep 15
+  fi
+fi
 
 echo "========================================="
 echo "== Setup environment specifics =========="
@@ -38,14 +46,9 @@ for filename in ${WHIRL_SETUP_FOLDER}/dag.d/*.sh; do
   fi
 done
 
-echo "Starting Airflow scheduler..."
-nohup airflow scheduler -D &
+if [[ ${AIRFLOW_COMMAND} == "webserver" || ${AIRFLOW_COMMAND} == "singlemachine" ]]; then
 
-echo  "wait a while for the scheduler to be started"
-sleep 15
-
-echo "If needed, unpause dags..."
-if [ "${UNPAUSE_DAG}" = true ]; then
+  if [ "${UNPAUSE_DAG}" = true ]; then
     echo "================================="
     echo "== Enabling all available DAGs =="
     echo "================================="
@@ -57,8 +60,14 @@ if [ "${UNPAUSE_DAG}" = true ]; then
       echo "Enabling DAG ${d}"
       airflow unpause "${d}" || true
     done
-  end
+  fi
 fi
 
-echo "Starting Airflow webserver..."
-airflow webserver -p 5000
+if [[ ${AIRFLOW_COMMAND} == "singlemachine" ]]; then
+  nohup /entrypoint scheduler -D &
+  # echo  "wait a while for the scheduler to be started"
+  # sleep 15
+  /entrypoint webserver -p 5000
+else
+  /entrypoint "${@}"
+fi

docker/airflow-python/includes/date_replacement.sh

@@ -44,7 +44,7 @@ function replace_date() {
   echo $RENAMES
 }
 
-SCHEDULE=$(grep -oP "schedule_interval=\K[^,]*" /usr/local/airflow/dags/*/*.py | head -n1 | sed -e "s/['\"]//g")
+SCHEDULE=$(grep -oP "schedule_interval=\K[^,]*" /opt/airflow/dags/*/*.py | head -n1 | sed -e "s/['\"]//g")
 
 # Change Airflow schedule annotations to crontab schedule values
 if [[ ${SCHEDULE} == *"@"* ]]; then

docker/aws-spark/Dockerfile

@@ -2,21 +2,21 @@ ARG SPARK_VERSION=latest
 FROM godatadriven/spark:${SPARK_VERSION}
 
 ENV POSTGRES_JDBC_CHECKSUM=7ffa46f8c619377cdebcd17721b6b21ecf6659850179f96fec3d1035cf5a0cdc
-ENV HADOOP_AWS_CHECKSUM=af9f18a0fcef4c564deea6f3ca1eec040b59be3d1cfd7fa557975d25d90e23f6
+ENV HADOOP_AWS_CHECKSUM=acf05db5e92f79b287444c9e6bd71f27f125193c47ef59149460ef02ef73a72c
 ENV AWS_SDK_CHECKSUM=ab74b9bd8baf700bbb8c1270c02d87e570cd237af2464bafa9db87ca1401143a
 
 RUN apt-get update && \
     apt-get install -y curl && \
     apt-get clean
 
-RUN curl -o /usr/spark/jars/aws-java-sdk-1.7.4.jar http://central.maven.org/maven2/com/amazonaws/aws-java-sdk/1.7.4/aws-java-sdk-1.7.4.jar && \
+RUN curl -o /usr/spark/jars/aws-java-sdk-1.7.4.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/1.7.4/aws-java-sdk-1.7.4.jar && \
   echo "$AWS_SDK_CHECKSUM /usr/spark/jars/aws-java-sdk-1.7.4.jar" | sha256sum -c -
 
 RUN curl -o /usr/spark/jars/postgresql-42.2.5.jar https://jdbc.postgresql.org/download/postgresql-42.2.5.jar && \
   echo "$POSTGRES_JDBC_CHECKSUM /usr/spark/jars/postgresql-42.2.5.jar" | sha256sum -c -
 
-RUN curl -o /usr/spark/jars/hadoop-aws-2.7.3.jar http://central.maven.org/maven2/org/apache/hadoop/hadoop-aws/2.7.3/hadoop-aws-2.7.3.jar && \
-  echo "$HADOOP_AWS_CHECKSUM /usr/spark/jars/hadoop-aws-2.7.3.jar" | sha256sum -c -
+RUN curl -o /usr/spark/jars/hadoop-aws-2.7.4.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/2.7.4/hadoop-aws-2.7.4.jar && \
+  echo "$HADOOP_AWS_CHECKSUM /usr/spark/jars/hadoop-aws-2.7.4.jar" | sha256sum -c -
 
 RUN echo "spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem" > ${SPARK_HOME}/conf/spark-defaults.conf && \
   echo "spark.hadoop.fs.s3a.connection.ssl.enabled=false" >> ${SPARK_HOME}/conf/spark-defaults.conf && \

envs/airflow-s3-logging/.whirl.env

@@ -8,4 +8,5 @@ PORT_WEB_UI=8080
 # Airflow variables
 AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
+AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False

envs/airflow-s3-logging/docker-compose.yml

@@ -3,6 +3,7 @@ version: '3'
 services:
   airflow:
     image: docker-whirl-airflow:py-${PYTHON_VERSION}-local
+    command: ["singlemachine"]
     ports:
       - '5000:5000'  # HTTP (Airflow Web UI)
     env_file:
@@ -11,7 +12,7 @@ services:
       - WHIRL_SETUP_FOLDER
       - UNPAUSE_DAG
     volumes:
-      - ${DAG_FOLDER}:/usr/local/airflow/dags/$PROJECTNAME
+      - ${DAG_FOLDER}:/opt/airflow/dags/$PROJECTNAME
       - ${ENVIRONMENT_FOLDER}/whirl.setup.d:${WHIRL_SETUP_FOLDER}/env.d/
       - ${DAG_FOLDER}/whirl.setup.d:${WHIRL_SETUP_FOLDER}/dag.d/
     depends_on:

envs/api-python-s3/.whirl.env

@@ -8,4 +8,5 @@ PORT_WEB_UI=8080
 # Airflow env vars
 AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
+AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False

envs/api-python-s3/docker-compose.yml

@@ -3,6 +3,7 @@ version: '3'
 services:
   airflow:
     image: docker-whirl-airflow:py-${PYTHON_VERSION}-local
+    command: ["singlemachine"]
     ports:
       - '5000:5000'  # HTTP (Airflow Web UI)
     env_file:
@@ -11,7 +12,7 @@ services:
       - WHIRL_SETUP_FOLDER
       - UNPAUSE_DAG
     volumes:
-      - ${DAG_FOLDER}:/usr/local/airflow/dags/$PROJECTNAME
+      - ${DAG_FOLDER}:/opt/airflow/dags/$PROJECTNAME
       - ${ENVIRONMENT_FOLDER}/whirl.setup.d:${WHIRL_SETUP_FOLDER}/env.d/
       - ${DAG_FOLDER}/whirl.setup.d:${WHIRL_SETUP_FOLDER}/dag.d/
     depends_on:
@@ -21,7 +22,7 @@ services:
       - s3server:${DEMO_BUCKET}.s3server
 
   mockserver:
-    image: jamesdbloom/mockserver:mockserver-5.6.1
+    image: jamesdbloom/mockserver:mockserver-5.11.1
     ports:
       - 1080:1080
       - 1081:1081

envs/external-airflow-db/.whirl.env

@@ -1,13 +1,14 @@
 # postgres env vars
 POSTGRES_HOST=postgresdb
 POSTGRES_PORT=5432
-POSTGRES_PASSWORD=p@ssw0rd
+POSTGRES_PASSWORD=pAssw0rd
 POSTGRES_USER=airflow
 POSTGRES_DB=airflow
 
 # Airflow variables
 AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
+AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False
 AIRFLOW__CORE__FERNET_KEY=W5gmA+dp84hkZEzpxPw4LTmhbXA1uVxKZsgIfay8wno=
 AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}

envs/external-airflow-db/docker-compose.yml

@@ -1,8 +1,9 @@
 version: '3'
 
 services:
-  airflow:
+  webserver:
     image: docker-whirl-airflow:py-${PYTHON_VERSION}-local
+    command: ["webserver", "-p", "5000"]
     ports:
       - '5000:5000'  # HTTP (Airflow Web UI)
     env_file:
@@ -11,14 +12,29 @@ services:
       - WHIRL_SETUP_FOLDER
       - UNPAUSE_DAG
     volumes:
-      - ${DAG_FOLDER}:/usr/local/airflow/dags/$PROJECTNAME
+      - ${DAG_FOLDER}:/opt/airflow/dags/$PROJECTNAME
+      - ${ENVIRONMENT_FOLDER}/whirl.setup.d:${WHIRL_SETUP_FOLDER}/env.d/
+      - ${DAG_FOLDER}/whirl.setup.d:${WHIRL_SETUP_FOLDER}/dag.d/
+    depends_on:
+      - postgresdb
+
+  scheduler:
+    image: docker-whirl-airflow:py-${PYTHON_VERSION}-local
+    command: ["scheduler"]
+    env_file:
+      - .whirl.env
+    environment:
+      - WHIRL_SETUP_FOLDER
+      - UNPAUSE_DAG
+    volumes:
+      - ${DAG_FOLDER}:/opt/airflow/dags/$PROJECTNAME
       - ${ENVIRONMENT_FOLDER}/whirl.setup.d:${WHIRL_SETUP_FOLDER}/env.d/
       - ${DAG_FOLDER}/whirl.setup.d:${WHIRL_SETUP_FOLDER}/dag.d/
     depends_on:
       - postgresdb
 
   postgresdb:
-    image: postgres:11
+    image: postgres:13
     ports:
       - 5432:5432
     environment:

envs/external-smtp-config/.whirl.env

@@ -1,6 +1,7 @@
 # Airflow variables
 AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
+AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False
 AIRFLOW__SMTP__SMTP_HOST=smtp-server
 AIRFLOW__SMTP__SMTP_STARTTLS=False

envs/external-smtp-config/docker-compose.yml

@@ -3,6 +3,7 @@ version: '3'
 services:
   airflow:
     image: docker-whirl-airflow:py-${PYTHON_VERSION}-local
+    command: ["singlemachine"]
     ports:
       - '5000:5000'  # HTTP (Airflow Web UI)
     env_file:
@@ -11,7 +12,7 @@ services:
       - WHIRL_SETUP_FOLDER
       - UNPAUSE_DAG
     volumes:
-      - ${DAG_FOLDER}:/usr/local/airflow/dags/$PROJECTNAME
+      - ${DAG_FOLDER}:/opt/airflow/dags/$PROJECTNAME
       - ${ENVIRONMENT_FOLDER}/whirl.setup.d:${WHIRL_SETUP_FOLDER}/env.d/
       - ${DAG_FOLDER}/whirl.setup.d:${WHIRL_SETUP_FOLDER}/dag.d/
     depends_on:

envs/local-ssh/.whirl.env

@@ -1,4 +1,5 @@
 # Airflow env vars
 AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
+AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False

envs/local-ssh/docker-compose.yml

@@ -3,6 +3,7 @@ version: '3'
 services:
   airflow:
     image: docker-whirl-airflow:py-${PYTHON_VERSION}-local
+    command: ["singlemachine"]
     ports:
       - '5000:5000'  # HTTP (Airflow Web UI)
     env_file:
@@ -11,6 +12,6 @@ services:
       - WHIRL_SETUP_FOLDER
       - UNPAUSE_DAG
     volumes:
-      - ${DAG_FOLDER}:/usr/local/airflow/dags/$PROJECTNAME
+      - ${DAG_FOLDER}:/opt/airflow/dags/$PROJECTNAME
       - ${ENVIRONMENT_FOLDER}/whirl.setup.d:${WHIRL_SETUP_FOLDER}/env.d/
       - ${DAG_FOLDER}/whirl.setup.d:${WHIRL_SETUP_FOLDER}/dag.d/

envs/postgres-s3-external-spark/.whirl.env

@@ -13,9 +13,10 @@ POSTGRES_USER=postgres
 POSTGRES_DB=postgresdb
 
 # Spark variables
-SPARK_VERSION=2.4.1
+SPARK_VERSION=3.0
 
 # Airflow env vars
 AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
+AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False