diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..ede15af0c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.github/* +.tx/* +conf/* +doc/* +envs/* +res/* +tools/* +.gitignore +.travis +Procfile +run.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..7758f4c7c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +FROM python:3.9-buster as base + +EXPOSE 8000 +STOPSIGNAL SIGTERM +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 +# ENV DEBUG 1 + +RUN apt-get update +RUN apt-get install -y libpq-dev libjpeg-dev zlib1g-dev libwebp-dev libffi-dev + +# copy source and install dependencies +RUN mkdir -p /app + +WORKDIR /app + +COPY . /app +RUN python -m pip install -r requirements.txt + +FROM base as dev + +RUN python -m pip install -r requirements-dev.txt +RUN make dev-config + +CMD ["envdir", "envs/dev", "python", "manage.py", "runserver", "0.0.0.0:8000"] + +FROM dev as test + +RUN python -m pip install -r requirements-test.txt + +CMD ["make", "test"] + +FROM base as production + +RUN python -m pip install -r requirements-kubernetes.txt + +CMD ["python", "-m", "kubernetes_wsgi", "mygpo.wsgi", "--port", "8000"] diff --git a/doc/dev/index.rst b/doc/dev/index.rst index d148bc9de..f3c72fffa 100644 --- a/doc/dev/index.rst +++ b/doc/dev/index.rst @@ -27,6 +27,7 @@ Contents :maxdepth: 1 installation + kubernetes postgres-setup libraries configuration diff --git a/doc/dev/installation.rst b/doc/dev/installation.rst index f3016a777..e7f3981df 100644 --- a/doc/dev/installation.rst +++ b/doc/dev/installation.rst @@ -168,3 +168,32 @@ directory with If you want to run a production server, check out `Deploying Django `_. + +Running with Docker +------------------- + +There is a multi-stage docker definition. To build the image for local testing run + +.. code-block:: bash + + docker build --target dev -t gpodder.dev -f Dockerfile . + +This will build the `dev` stage. + +Next, you need to define the configuration to be passed into the container. The simplest is to run + +.. code-block:: bash + + make dev-config + +Next, you need to run the migrations: + +.. code-block:: bash + + docker run -ti -v ${PWD}/envs/dev:/app/envs/dev gpodder.dev envdir envs/dev python manage.py migrate + +Finally, you can start the development server: + +.. code-block:: bash + + docker run --rm -ti -v ${PWD}/envs/dev:/app/envs/dev gpodder.dev diff --git a/doc/dev/kubernetes.rst b/doc/dev/kubernetes.rst new file mode 100644 index 000000000..114d7a9de --- /dev/null +++ b/doc/dev/kubernetes.rst @@ -0,0 +1,212 @@ +Deploying to Kubernetes +======================== + +The provided docker file is ready to be built and deployed to Kubernetes. + +At this stage, the project does not provide pre-build containers, but PR to add it to GitHub actions are welcome. + +This section assumes that you built the `production` docker layer and have it in your container registry and you know how to deploy your YAML manifests to Kubernetes (e.g with FluxCD). + +There are many ways to store the necessary secrets, configuration and everything in Kubernetes. This guide provides the bare minimum. + +Configurations +--------------- + +```yaml +# Based on https://gpoddernet.readthedocs.io/en/latest/dev/configuration.html#configuration +apiVersion: v1 +kind: ConfigMap +metadata: + name: gpodder-app-config +data: + DJANGO_CONFIGURATION: Prod + ADMINS: 'Your Name ' + ALLOWED_HOSTS: '*' + DEFAULT_BASE_URL: 'https://gpodder.example.com' + # GOOGLE_ANALYTICS_PROPERTY_ID + # MAINTENANCE + DEBUG: "true" + DEFAULT_FROM_EMAIL: "daemon@example.com" + SERVER_EMAIL: "daemon@example.com" + BROKER_POOL_LIMIT: "10" + # CACHE_BACKEND: "django.core.cache.backends.db.DatabaseCache" + # ACCOUNT_ACTIVATION_DAYS + PODCAST_SLUG_SUBSCRIBER_LIMIT: "1" + MIN_SUBSCRIBERS_CATEGORY: "1" + # INTERNAL_IPS: +``` + +Secrets +-------- + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: gpodder +spec: + encryptedData: + BROKER_URL: + EMAIL_HOST: + EMAIL_HOST_PASSWORD: + EMAIL_HOST_USER: + SECRET_KEY: + STAFF_TOKEN: + SUPPORT_URL: + DATABASE_URL: + AWS_ACCESS_KEY_ID: + AWS_S3_ENDPOINT_URL: + AWS_S3_ENDPOINT_URL: + +Jobs +---- + +As Kubernetes Jobs are immutable. It's up to you how you re-run them on changes. This guide does not help with it. A possible approach [using FluxCD is described here](https://fluxcd.io/flux/use-cases/running-jobs/. + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: collectstatics + labels: + app.kubernetes.io/component: collectstatics +spec: + template: + spec: + serviceAccountName: gpodder + containers: + - name: gpodder-migrate + image: registry.gitlab.com/nagyv/gpodder/gpodder:latest + command: ["python", "manage.py", "collectstatic", "--no-input"] + envFrom: + - secretRef: + name: gpodder + - configMapRef: + name: gpodder-app-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + restartPolicy: Never +``` + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: gpodder-migrate + labels: + app.kubernetes.io/component: db-migrate +spec: + # ttlSecondsAfterFinished does not work well with GitOps as the Job is deleted + # ttlSecondsAfterFinished: 200 + template: + metadata: + labels: + app.kubernetes.io/component: db-migrate + spec: + serviceAccountName: gpodder + containers: + - name: gpodder-migrate + image: registry.gitlab.com/nagyv/gpodder/gpodder:latest + command: ["python", "manage.py", "migrate"] + envFrom: + - secretRef: + name: gpodder + - configMapRef: + name: gpodder-app-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + restartPolicy: Never +``` + +Deployment +----------- + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gpodder + labels: + app.kubernetes.io/component: webapp +spec: + selector: + matchLabels: + app.kubernetes.io/component: webapp + template: + metadata: + labels: + app.kubernetes.io/component: webapp + spec: + serviceAccountName: gpodder + containers: + - name: gpodder + image: registry.example.com/gpodder/gpodder:latest + imagePullPolicy: Always + resources: {} + # limits: + # memory: "128Mi" + # cpu: "500m" + # livenessProbe: + # httpGet: + # path: /ht/ + # port: 8000 + # httpHeaders: + # - name: Host + # value: gpodder.nagyv.com + # initialDelaySeconds: 15 + # periodSeconds: 10 + # successThreshold: 1 + # failureThreshold: 2 + # timeoutSeconds: 3 + readinessProbe: + httpGet: + path: /ht/ + port: 8000 + httpHeaders: + - name: Host + value: gpodder.nagyv.com + initialDelaySeconds: 10 + timeoutSeconds: 3 + ports: + - containerPort: 8000 + envFrom: + - secretRef: + name: gpodder + - configMapRef: + name: gpodder-app-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + securityContext: + {} +``` + +Service +------- + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: gpodder +spec: + selector: {} + ports: + - port: 80 + targetPort: 8000 + protocol: TCP +``` diff --git a/manage.py b/manage.py index 81d625f10..608b67dbc 100644 --- a/manage.py +++ b/manage.py @@ -1,9 +1,19 @@ import os import sys -if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mygpo.settings") - - from django.core.management import execute_from_command_line - +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mygpo.settings') + os.environ.setdefault('DJANGO_CONFIGURATION', 'Base') + try: + from configurations.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc execute_from_command_line(sys.argv) + +if __name__ == '__main__': + main() diff --git a/mygpo/asgi.py b/mygpo/asgi.py index e125fe830..3cd31cb04 100644 --- a/mygpo/asgi.py +++ b/mygpo/asgi.py @@ -9,8 +9,9 @@ import os -from django.core.asgi import get_asgi_application +from configurations.asgi import get_asgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mygpo.settings") +os.environ.setdefault('DJANGO_CONFIGURATION', 'Prod') application = get_asgi_application() diff --git a/mygpo/celery.py b/mygpo/celery.py index 86634da78..a763871a2 100644 --- a/mygpo/celery.py +++ b/mygpo/celery.py @@ -5,6 +5,10 @@ from celery import Celery os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mygpo.settings") +os.environ.setdefault('DJANGO_CONFIGURATION', 'Prod') + +import configurations +configurations.setup() celery = Celery("mygpo.celery") celery.config_from_object("django.conf:settings", namespace="CELERY") diff --git a/mygpo/settings.py b/mygpo/settings.py index b39273153..2c655d729 100644 --- a/mygpo/settings.py +++ b/mygpo/settings.py @@ -14,6 +14,7 @@ import django import six +from configurations import Configuration django.utils.six = six @@ -32,387 +33,428 @@ def get_intOrNone(name, default): return int(value) -DEBUG = get_bool("DEBUG", False) +class BaseConfig(Configuration): -ADMINS = re.findall(r"\s*([^<]+) <([^>]+)>\s*", os.getenv("ADMINS", "")) + DEBUG = get_bool("DEBUG", False) -MANAGERS = ADMINS + ADMINS = re.findall(r"\s*([^<]+) <([^>]+)>\s*", os.getenv("ADMINS", "")) -DATABASES = { - "default": dj_database_url.config(default="postgres://mygpo:mygpo@localhost/mygpo") -} + MANAGERS = ADMINS - -_USE_GEVENT = get_bool("USE_GEVENT", False) -if _USE_GEVENT: - # see https://github.com/jneight/django-db-geventpool - default = DATABASES["default"] - default["ENGINE"] = ("django_db_geventpool.backends.postgresql_psycopg2",) - default["CONN_MAX_AGE"] = 0 - options = default.get("OPTIONS", {}) - options["MAX_CONNS"] = 20 - - -_cache_used = bool(os.getenv("CACHE_BACKEND", False)) - -if _cache_used: - CACHES = {} - CACHES["default"] = { - "BACKEND": os.getenv( - "CACHE_BACKEND", "django.core.cache.backends.memcached.MemcachedCache" - ), - "LOCATION": os.getenv("CACHE_LOCATION"), + _DATABASES = { + "default": dj_database_url.config(default="postgres://mygpo:mygpo@localhost/mygpo") } + _USE_GEVENT = get_bool("USE_GEVENT", False) + @property + def DATABASES(self): + if self._USE_GEVENT: + # see https://github.com/jneight/django-db-geventpool + default = self._DATABASES["default"] + default["ENGINE"] = ("django_db_geventpool.backends.postgresql_psycopg2",) + default["CONN_MAX_AGE"] = 0 + options = default.get("OPTIONS", {}) + options["MAX_CONNS"] = 20 + return self._DATABASES -# Local time zone for this installation. Choices can be found here: -# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name -# although not all choices may be available on all operating systems. -# If running in a Windows environment this must be set to the same as your -# system time zone. -TIME_ZONE = "UTC" - -# Language code for this installation. All choices can be found here: -# http://www.i18nguy.com/unicode/language-identifiers.html -LANGUAGE_CODE = "en-us" - -SITE_ID = 1 - -# If you set this to False, Django will make some optimizations so as not -# to load the internationalization machinery. -USE_I18N = True - - -# Static Files - -STATIC_ROOT = "staticfiles" -STATIC_URL = "/static/" - -STATICFILES_DIRS = (os.path.abspath(os.path.join(BASE_DIR, "..", "static")),) - - -# Media Files - -MEDIA_ROOT = os.getenv( - "MEDIA_ROOT", os.path.abspath(os.path.join(BASE_DIR, "..", "media")) -) - -MEDIA_URL = "/media/" - - -TEMPLATES = [ - { - "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [], - "OPTIONS": { - "debug": DEBUG, - "context_processors": [ - "django.contrib.auth.context_processors.auth", - "django.template.context_processors.debug", - "django.template.context_processors.i18n", - "django.template.context_processors.media", - "django.template.context_processors.static", - "django.template.context_processors.tz", - "django.contrib.messages.context_processors.messages", - "mygpo.web.google.analytics", - "mygpo.web.google.adsense", - # make the debug variable available in templates - # https://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-debug - "django.template.context_processors.debug", - # required so that the request obj can be accessed from - # templates. this is used to direct users to previous - # page after login - "django.template.context_processors.request", - ], - "libraries": {"staticfiles": "django.templatetags.static"}, - "loaders": [ - ( - "django.template.loaders.cached.Loader", - ["django.template.loaders.app_directories.Loader"], - ) - ], - }, - } -] - - -MIDDLEWARE = [ - "django.middleware.common.CommonMiddleware", - "django.middleware.csrf.CsrfViewMiddleware", - "django.contrib.sessions.middleware.SessionMiddleware", - "django.contrib.auth.middleware.AuthenticationMiddleware", - "django.middleware.locale.LocaleMiddleware", - "django.contrib.messages.middleware.MessageMiddleware", -] - -ROOT_URLCONF = "mygpo.urls" - -INSTALLED_APPS = [ - "django.contrib.contenttypes", - "django.contrib.messages", - "django.contrib.admin", - "django.contrib.humanize", - "django.contrib.auth", - "django.contrib.sessions", - "django.contrib.staticfiles", - "django.contrib.sites", - "django.contrib.postgres", - "django_celery_results", - "django_celery_beat", - "mygpo.core", - "mygpo.podcasts", - "mygpo.chapters", - "mygpo.search", - "mygpo.users", - "mygpo.api", - "mygpo.web", - "mygpo.publisher", - "mygpo.subscriptions", - "mygpo.history", - "mygpo.favorites", - "mygpo.usersettings", - "mygpo.data", - "mygpo.userfeeds", - "mygpo.suggestions", - "mygpo.directory", - "mygpo.categories", - "mygpo.episodestates", - "mygpo.maintenance", - "mygpo.share", - "mygpo.administration", - "mygpo.pubsub", - "mygpo.podcastlists", - "mygpo.votes", -] + _cache_used = bool(os.getenv("CACHE_BACKEND", False)) -try: - if DEBUG: - import debug_toolbar + @property + def CACHES(self): + # Set the default django backend as a fallback + CACHES = Configuration.CACHES + if self._cache_used: + CACHES["default"] = { + "BACKEND": os.getenv( + "CACHE_BACKEND", "django.core.cache.backends.memcached.MemcachedCache" + ), + "LOCATION": os.getenv("CACHE_LOCATION"), + } + return CACHES - INSTALLED_APPS += ["debug_toolbar"] - MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"] + # Local time zone for this installation. Choices can be found here: + # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name + # although not all choices may be available on all operating systems. + # If running in a Windows environment this must be set to the same as your + # system time zone. + TIME_ZONE = "UTC" -except ImportError: - pass + # Language code for this installation. All choices can be found here: + # http://www.i18nguy.com/unicode/language-identifiers.html + LANGUAGE_CODE = "en-us" + SITE_ID = 1 -try: - if DEBUG: - import django_extensions + # If you set this to False, Django will make some optimizations so as not + # to load the internationalization machinery. + USE_I18N = True - INSTALLED_APPS += ["django_extensions"] -except ImportError: - pass + # Static Files + STATIC_ROOT = "staticfiles" + STATIC_URL = "/static/" -ACCOUNT_ACTIVATION_DAYS = int(os.getenv("ACCOUNT_ACTIVATION_DAYS", 7)) + STATICFILES_DIRS = (os.path.abspath(os.path.join(BASE_DIR, "..", "static")),) -AUTHENTICATION_BACKENDS = ( - "mygpo.users.backend.CaseInsensitiveModelBackend", - "mygpo.web.auth.EmailAuthenticationBackend", -) -SESSION_ENGINE = "django.contrib.sessions.backends.cached_db" + # Media Files -# TODO: use (default) JSON serializer for security -# this would currently fail as we're (de)serializing datetime objects -# https://docs.djangoproject.com/en/1.5/topics/http/sessions/#session-serialization -SESSION_SERIALIZER = "django.contrib.sessions.serializers.PickleSerializer" + MEDIA_ROOT = os.getenv( + "MEDIA_ROOT", os.path.abspath(os.path.join(BASE_DIR, "..", "media")) + ) + MEDIA_URL = "/media/" + + TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "OPTIONS": { + "debug": DEBUG, + "context_processors": [ + "django.contrib.auth.context_processors.auth", + "django.template.context_processors.debug", + "django.template.context_processors.i18n", + "django.template.context_processors.media", + "django.template.context_processors.static", + "django.template.context_processors.tz", + "django.contrib.messages.context_processors.messages", + "mygpo.web.google.analytics", + "mygpo.web.google.adsense", + # make the debug variable available in templates + # https://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-debug + "django.template.context_processors.debug", + # required so that the request obj can be accessed from + # templates. this is used to direct users to previous + # page after login + "django.template.context_processors.request", + ], + "libraries": {"staticfiles": "django.templatetags.static"}, + "loaders": [ + ( + "django.template.loaders.cached.Loader", + ["django.template.loaders.app_directories.Loader"], + ) + ], + }, + } + ] + + + MIDDLEWARE = [ + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.middleware.locale.LocaleMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + ] + + ROOT_URLCONF = "mygpo.urls" + + INSTALLED_APPS = [ + "django.contrib.contenttypes", + "django.contrib.messages", + "django.contrib.admin", + "django.contrib.humanize", + "django.contrib.auth", + "django.contrib.sessions", + "django.contrib.staticfiles", + "django.contrib.sites", + "django.contrib.postgres", + "django_celery_results", + "django_celery_beat", + "mygpo.core", + "mygpo.podcasts", + "mygpo.chapters", + "mygpo.search", + "mygpo.users", + "mygpo.api", + "mygpo.web", + "mygpo.publisher", + "mygpo.subscriptions", + "mygpo.history", + "mygpo.favorites", + "mygpo.usersettings", + "mygpo.data", + "mygpo.userfeeds", + "mygpo.suggestions", + "mygpo.directory", + "mygpo.categories", + "mygpo.episodestates", + "mygpo.maintenance", + "mygpo.share", + "mygpo.administration", + "mygpo.pubsub", + "mygpo.podcastlists", + "mygpo.votes", + ] + + ACCOUNT_ACTIVATION_DAYS = int(os.getenv("ACCOUNT_ACTIVATION_DAYS", 7)) + + AUTHENTICATION_BACKENDS = ( + "mygpo.users.backend.CaseInsensitiveModelBackend", + "mygpo.web.auth.EmailAuthenticationBackend", + ) -MESSAGE_STORAGE = "django.contrib.messages.storage.session.SessionStorage" + SESSION_ENGINE = "django.contrib.sessions.backends.cached_db" -USER_CLASS = "mygpo.users.models.User" + # TODO: use (default) JSON serializer for security + # this would currently fail as we're (de)serializing datetime objects + # https://docs.djangoproject.com/en/1.5/topics/http/sessions/#session-serialization + SESSION_SERIALIZER = "django.contrib.sessions.serializers.PickleSerializer" -LOGIN_URL = "/login/" -CSRF_FAILURE_VIEW = "mygpo.web.views.csrf_failure" + MESSAGE_STORAGE = "django.contrib.messages.storage.session.SessionStorage" + USER_CLASS = "mygpo.users.models.User" -DEFAULT_FROM_EMAIL = os.getenv("DEFAULT_FROM_EMAIL", "") + LOGIN_URL = "/login/" -SERVER_EMAIL = os.getenv("SERVER_EMAIL", DEFAULT_FROM_EMAIL) + CSRF_FAILURE_VIEW = "mygpo.web.views.csrf_failure" -SECRET_KEY = os.getenv("SECRET_KEY", "") -if "pytest" in sys.argv[0]: - SECRET_KEY = "test" + DEFAULT_FROM_EMAIL = os.getenv("DEFAULT_FROM_EMAIL", "") -GOOGLE_ANALYTICS_PROPERTY_ID = os.getenv("GOOGLE_ANALYTICS_PROPERTY_ID", "") + SERVER_EMAIL = os.getenv("SERVER_EMAIL", DEFAULT_FROM_EMAIL) -DIRECTORY_EXCLUDED_TAGS = os.getenv("DIRECTORY_EXCLUDED_TAGS", "").split() + SECRET_KEY = os.getenv("SECRET_KEY", "") -FLICKR_API_KEY = os.getenv("FLICKR_API_KEY", "") + GOOGLE_ANALYTICS_PROPERTY_ID = os.getenv("GOOGLE_ANALYTICS_PROPERTY_ID", "") -SOUNDCLOUD_CONSUMER_KEY = os.getenv("SOUNDCLOUD_CONSUMER_KEY", "") + DIRECTORY_EXCLUDED_TAGS = os.getenv("DIRECTORY_EXCLUDED_TAGS", "").split() -MAINTENANCE = get_bool("MAINTENANCE", False) + FLICKR_API_KEY = os.getenv("FLICKR_API_KEY", "") + SOUNDCLOUD_CONSUMER_KEY = os.getenv("SOUNDCLOUD_CONSUMER_KEY", "") -ALLOWED_HOSTS = ["*"] + MAINTENANCE = get_bool("MAINTENANCE", False) + ALLOWED_HOSTS = ["*"] -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "verbose": {"format": "%(asctime)s %(name)s %(levelname)s %(message)s"} - }, - "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}}, - "handlers": { - "console": { - "level": os.getenv("LOGGING_CONSOLE_LEVEL", "DEBUG"), - "class": "logging.StreamHandler", - "formatter": "verbose", - }, - "mail_admins": { - "level": "ERROR", - "filters": ["require_debug_false"], - "class": "django.utils.log.AdminEmailHandler", + _LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": {"format": "%(asctime)s %(name)s %(levelname)s %(message)s"} }, - }, - "loggers": { - "django": { - "handlers": os.getenv("LOGGING_DJANGO_HANDLERS", "console").split(), - "propagate": True, - "level": os.getenv("LOGGING_DJANGO_LEVEL", "WARN"), + "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}}, + "handlers": { + "console": { + "level": os.getenv("LOGGING_CONSOLE_LEVEL", "DEBUG"), + "class": "logging.StreamHandler", + "formatter": "verbose", + }, + "mail_admins": { + "level": "ERROR", + "filters": ["require_debug_false"], + "class": "django.utils.log.AdminEmailHandler", + }, }, - "mygpo": { - "handlers": os.getenv("LOGGING_MYGPO_HANDLERS", "console").split(), - "level": os.getenv("LOGGING_MYGPO_LEVEL", "INFO"), + "loggers": { + "django": { + "handlers": os.getenv("LOGGING_DJANGO_HANDLERS", "console").split(), + "propagate": True, + "level": os.getenv("LOGGING_DJANGO_LEVEL", "WARN"), + }, + "mygpo": { + "handlers": os.getenv("LOGGING_MYGPO_HANDLERS", "console").split(), + "level": os.getenv("LOGGING_MYGPO_LEVEL", "INFO"), + }, + "celery": { + "handlers": os.getenv("LOGGING_CELERY_HANDLERS", "console").split(), + "level": os.getenv("LOGGING_CELERY_LEVEL", "DEBUG"), + }, }, - "celery": { - "handlers": os.getenv("LOGGING_CELERY_HANDLERS", "console").split(), - "level": os.getenv("LOGGING_CELERY_LEVEL", "DEBUG"), - }, - }, -} - -_use_log_file = bool(os.getenv("LOGGING_FILENAME", False)) - -if _use_log_file: - LOGGING["handlers"]["file"] = { - "level": "INFO", - "class": "logging.handlers.RotatingFileHandler", - "filename": os.getenv("LOGGING_FILENAME"), - "maxBytes": 10_000_000, - "backupCount": 10, - "formatter": "verbose", } + _use_log_file = bool(os.getenv("LOGGING_FILENAME", False)) -DATA_UPLOAD_MAX_MEMORY_SIZE = get_intOrNone("DATA_UPLOAD_MAX_MEMORY_SIZE", None) - - -# minimum number of subscribers a podcast must have to be assigned a slug -PODCAST_SLUG_SUBSCRIBER_LIMIT = int(os.getenv("PODCAST_SLUG_SUBSCRIBER_LIMIT", 10)) - -# minimum number of subscribers that a podcast needs to "push" one of its -# categories to the top -MIN_SUBSCRIBERS_CATEGORY = int(os.getenv("MIN_SUBSCRIBERS_CATEGORY", 10)) - -# maximum number of episode actions that the API processes immediatelly before -# returning the response. Larger requests will be handled in background. -# Handler can be set to None to disable -API_ACTIONS_MAX_NONBG = get_intOrNone("API_ACTIONS_MAX_NONBG", 100) -API_ACTIONS_BG_HANDLER = "mygpo.api.tasks.episode_actions_celery_handler" + @property + def LOGGING(self): + if self._use_log_file: + self._LOGGING["handlers"]["file"] = { + "level": "INFO", + "class": "logging.handlers.RotatingFileHandler", + "filename": os.getenv("LOGGING_FILENAME"), + "maxBytes": 10_000_000, + "backupCount": 10, + "formatter": "verbose", + } + return self._LOGGING + DATA_UPLOAD_MAX_MEMORY_SIZE = get_intOrNone("DATA_UPLOAD_MAX_MEMORY_SIZE", None) -ADSENSE_CLIENT = os.getenv("ADSENSE_CLIENT", "") + # minimum number of subscribers a podcast must have to be assigned a slug + PODCAST_SLUG_SUBSCRIBER_LIMIT = int(os.getenv("PODCAST_SLUG_SUBSCRIBER_LIMIT", 10)) -ADSENSE_SLOT_BOTTOM = os.getenv("ADSENSE_SLOT_BOTTOM", "") + # minimum number of subscribers that a podcast needs to "push" one of its + # categories to the top + MIN_SUBSCRIBERS_CATEGORY = int(os.getenv("MIN_SUBSCRIBERS_CATEGORY", 10)) -# we're running behind a proxy that sets the X-Forwarded-Proto header correctly -# see https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header -SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + # maximum number of episode actions that the API processes immediatelly before + # returning the response. Larger requests will be handled in background. + # Handler can be set to None to disable + API_ACTIONS_MAX_NONBG = get_intOrNone("API_ACTIONS_MAX_NONBG", 100) + API_ACTIONS_BG_HANDLER = "mygpo.api.tasks.episode_actions_celery_handler" + ADSENSE_CLIENT = os.getenv("ADSENSE_CLIENT", "") -# enabled access to staff-only areas with ?staff= -STAFF_TOKEN = os.getenv("STAFF_TOKEN", None) + ADSENSE_SLOT_BOTTOM = os.getenv("ADSENSE_SLOT_BOTTOM", "") -# The User-Agent string used for outgoing HTTP requests -USER_AGENT = "gpodder.net (+https://github.com/gpodder/mygpo)" + # we're running behind a proxy that sets the X-Forwarded-Proto header correctly + # see https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header + SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") -# Base URL of the website that is used if the actually used parameters is not -# available. Request handlers, for example, can access the requested domain. -# Code that runs in background can not do this, and therefore requires a -# default value. This should be set to something like 'http://example.com' -DEFAULT_BASE_URL = os.getenv("DEFAULT_BASE_URL", "") + # enabled access to staff-only areas with ?staff= + STAFF_TOKEN = os.getenv("STAFF_TOKEN", None) + # The User-Agent string used for outgoing HTTP requests + USER_AGENT = "gpodder.net (+https://github.com/gpodder/mygpo)" -### Celery + # Base URL of the website that is used if the actually used parameters is not + # available. Request handlers, for example, can access the requested domain. + # Code that runs in background can not do this, and therefore requires a + # default value. This should be set to something like 'http://example.com' + DEFAULT_BASE_URL = os.getenv("DEFAULT_BASE_URL", "") -CELERY_BROKER_URL = os.getenv("BROKER_URL", "redis://localhost") -CELERY_RESULT_BACKEND = "django-db" + ### Celery -CELERY_RESULT_EXPIRES = 60 * 60 # 1h expiry time in seconds + CELERY_BROKER_URL = os.getenv("BROKER_URL", "redis://localhost") + CELERY_RESULT_BACKEND = os.getenv("BROKER_BACKEND", "django-db") -CELERY_ACCEPT_CONTENT = ["json"] + CELERY_RESULT_EXPIRES = 60 * 60 # 1h expiry time in seconds + CELERY_ACCEPT_CONTENT = ["json"] -### Google API + ### Google API -GOOGLE_CLIENT_ID = os.getenv("GOOGLE_CLIENT_ID", "") -GOOGLE_CLIENT_SECRET = os.getenv("GOOGLE_CLIENT_SECRET", "") + GOOGLE_CLIENT_ID = os.getenv("GOOGLE_CLIENT_ID", "") + GOOGLE_CLIENT_SECRET = os.getenv("GOOGLE_CLIENT_SECRET", "") -# URL where users of the site can get support -SUPPORT_URL = os.getenv("SUPPORT_URL", "") + # URL where users of the site can get support + SUPPORT_URL = os.getenv("SUPPORT_URL", "") + FEEDSERVICE_URL = os.getenv("FEEDSERVICE_URL", "http://feeds.gpodder.net/") -FEEDSERVICE_URL = os.getenv("FEEDSERVICE_URL", "http://feeds.gpodder.net/") + # time for how long an activation is valid; after that, an unactivated user + # will be deleted + ACTIVATION_VALID_DAYS = int(os.getenv("ACTIVATION_VALID_DAYS", 10)) + OPBEAT = { + "ORGANIZATION_ID": os.getenv("OPBEAT_ORGANIZATION_ID", ""), + "APP_ID": os.getenv("OPBEAT_APP_ID", ""), + "SECRET_TOKEN": os.getenv("OPBEAT_SECRET_TOKEN", ""), + } -# time for how long an activation is valid; after that, an unactivated user -# will be deleted -ACTIVATION_VALID_DAYS = int(os.getenv("ACTIVATION_VALID_DAYS", 10)) + LOCALE_PATHS = [os.path.abspath(os.path.join(BASE_DIR, "locale"))] + INTERNAL_IPS = os.getenv("INTERNAL_IPS", "").split() -OPBEAT = { - "ORGANIZATION_ID": os.getenv("OPBEAT_ORGANIZATION_ID", ""), - "APP_ID": os.getenv("OPBEAT_APP_ID", ""), - "SECRET_TOKEN": os.getenv("OPBEAT_SECRET_TOKEN", ""), -} + EMAIL_BACKEND = os.getenv( + "EMAIL_BACKEND", "django.core.mail.backends.smtp.EmailBackend" + ) -LOCALE_PATHS = [os.path.abspath(os.path.join(BASE_DIR, "locale"))] + PODCAST_AD_ID = os.getenv("PODCAST_AD_ID") -INTERNAL_IPS = os.getenv("INTERNAL_IPS", "").split() + MAX_EPISODE_ACTIONS = int(os.getenv("MAX_EPISODE_ACTIONS", 1000)) -EMAIL_BACKEND = os.getenv( - "EMAIL_BACKEND", "django.core.mail.backends.smtp.EmailBackend" -) + SEARCH_CUTOFF = float(os.getenv("SEARCH_CUTOFF", 0.3)) -PODCAST_AD_ID = os.getenv("PODCAST_AD_ID") + # Maximum non-whitespace length of search query + # If length of query is shorter than QUERY_LENGTH_CUTOFF, no results + # will be returned to avoid a server timeout due to too many possible + # responses + QUERY_LENGTH_CUTOFF = int(os.getenv("QUERY_LENGTH_CUTOFF", 3)) -MAX_EPISODE_ACTIONS = int(os.getenv("MAX_EPISODE_ACTIONS", 1000)) +class Local(BaseConfig): -SEARCH_CUTOFF = float(os.getenv("SEARCH_CUTOFF", 0.3)) + @classmethod + def setup(cls): + super(BaseConfig, cls).setup() + if cls.DEBUG: + try: + import debug_toolbar + cls.INSTALLED_APPS += ["debug_toolbar"] + cls.MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"] + except ImportError: + pass + try: + import django_extensions + cls.INSTALLED_APPS += ["django_extensions"] + except ImportError: + pass -# Maximum non-whitespace length of search query -# If length of query is shorter than QUERY_LENGTH_CUTOFF, no results -# will be returned to avoid a server timeout due to too many possible -# responses -QUERY_LENGTH_CUTOFF = int(os.getenv("QUERY_LENGTH_CUTOFF", 3)) -### Sentry +class Test(BaseConfig): + SECRET_KEY = "test" -try: - import sentry_sdk - from sentry_sdk.integrations.django import DjangoIntegration - from sentry_sdk.integrations.celery import CeleryIntegration - from sentry_sdk.integrations.redis import RedisIntegration - - # Sentry Data Source Name (DSN) - sentry_dsn = os.getenv("SENTRY_DSN", "") - if not sentry_dsn: - raise ValueError("Could not set up sentry because " "SENTRY_DSN is not set") - - sentry_sdk.init( - dsn=sentry_dsn, - integrations=[DjangoIntegration(), CeleryIntegration(), RedisIntegration()], - send_default_pii=True, - ) -except (ImportError, ValueError): - pass +class StorageMixin(): + DEFAULT_FILE_STORAGE = 'mygpo.storages.MediaStorage' + STATICFILES_STORAGE = 'mygpo.storages.StaticStorage' + AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') + AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') + AWS_S3_ENDPOINT_URL = os.getenv('AWS_S3_ENDPOINT_URL', 'https://localhost:9000') + AWS_S3_VERIFY = True + AWS_S3_REGION_NAME = 'ams3' + AWS_DEFAULT_ACL = 'public-read' + AWS_QUERYSTRING_AUTH=False + MEDIA_URL = os.getenv('AWS_S3_ENDPOINT_URL', 'https://localhost:9000/') + 'uploads/' + STATIC_URL = os.getenv('AWS_S3_ENDPOINT_URL', 'https://localhost:9000/') + 'statics/' + + +class Prod(StorageMixin, BaseConfig): + ALLOWED_HOSTS = os.getenv("ALLOWED_HOSTS", "127.0.0.1,localhost").split(",") + SITE_ID=1 + DEBUG = os.getenv('DEBUG', False) + # SECURE_SSL_REDIRECT = True + SESSION_COOKIE_SECURE = True + CSRF_COOKIE_SECURE = True + SECURE_HSTS_SECONDS = 31536000 + SECURE_HSTS_INCLUDE_SUBDOMAINS = True + SECURE_HSTS_PRELOAD = True + EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' + EMAIL_HOST = os.getenv('EMAIL_HOST', '') + EMAIL_PORT = os.getenv('EMAIL_HOST_PORT', 587) + EMAIL_HOST_USER = os.getenv('EMAIL_HOST_USER', '') + EMAIL_HOST_PASSWORD = os.getenv('EMAIL_HOST_PASSWORD', '') + EMAIL_USE_TLS = True + + @property + def INSTALLED_APPS(self): + installed_apps = super().INSTALLED_APPS[:] + return installed_apps + [ + 'health_check', + 'health_check.db', + 'health_check.cache', + 'health_check.storage', + 'health_check.contrib.migrations', + ] + + @classmethod + def post_setup(cls): + """Sentry initialization""" + super(Prod, cls).post_setup() + ### Sentry + try: + import sentry_sdk + from sentry_sdk.integrations.django import DjangoIntegration + from sentry_sdk.integrations.celery import CeleryIntegration + from sentry_sdk.integrations.redis import RedisIntegration + + # Sentry Data Source Name (DSN) + sentry_dsn = os.getenv("SENTRY_DSN", "") + if not sentry_dsn: + raise ValueError("Could not set up sentry because " "SENTRY_DSN is not set") + + sentry_sdk.init( + dsn=sentry_dsn, + integrations=[DjangoIntegration(), CeleryIntegration(), RedisIntegration()], + send_default_pii=True, + ) + + except (ImportError, ValueError): + pass diff --git a/mygpo/storages.py b/mygpo/storages.py new file mode 100644 index 000000000..091af1ea9 --- /dev/null +++ b/mygpo/storages.py @@ -0,0 +1,9 @@ +import os +from storages.backends.s3boto3 import S3Boto3Storage +class MediaStorage(S3Boto3Storage): + bucket_name = 'gpodder-statics' + location = 'uploads' + +class StaticStorage(S3Boto3Storage): + bucket_name = 'gpodder-statics' + location = 'statics' diff --git a/mygpo/urls.py b/mygpo/urls.py index d99ae3b17..357297a73 100644 --- a/mygpo/urls.py +++ b/mygpo/urls.py @@ -18,13 +18,18 @@ urlpatterns += [re_path("", utils.maintenance)] # Add debug urlpattern for debug_toolbar -if settings.DEBUG: +if 'debug_toolbar' in settings.INSTALLED_APPS: import debug_toolbar urlpatterns += [ path("__debug__/", include(debug_toolbar.urls)), ] +if 'health_check' in settings.INSTALLED_APPS: + urlpatterns += [ + path('ht/', include('health_check.urls')), + ] + # URLs are still registered during maintenace mode because we need to # build links from them (eg login-link). urlpatterns += [ diff --git a/mygpo/wsgi.py b/mygpo/wsgi.py index c94936600..b9cf5fac0 100755 --- a/mygpo/wsgi.py +++ b/mygpo/wsgi.py @@ -5,7 +5,8 @@ # Set the DJANGO_SETTINGS_MODULE environment variable os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mygpo.settings") +os.environ.setdefault('DJANGO_CONFIGURATION', 'Prod') -from django.core.wsgi import get_wsgi_application +from configurations.wsgi import get_wsgi_application application = get_wsgi_application() diff --git a/requirements-kubernetes.txt b/requirements-kubernetes.txt new file mode 100644 index 000000000..2511ad550 --- /dev/null +++ b/requirements-kubernetes.txt @@ -0,0 +1,5 @@ +sentry-sdk==1.9.10 +kubernetes-wsgi>=1.0,<2.0 +django-storages>=1.11.1,<2.0 +django-health-check>=3.16.5,<4.0 +boto3>=1.17.85,<2.0 diff --git a/requirements.txt b/requirements.txt index 225d28973..eec0b8f49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +envdir Django==3.2.14 celery==5.2.7 Babel==2.10.3 @@ -18,3 +19,4 @@ django-celery-beat==2.3.0 django-celery-results==2.4.0 requests==2.28.1 django-db-geventpool==4.0.1 +django-configurations>=2.2,<3.0 \ No newline at end of file