From b17a4d95b7424272838519ac353d05d357c18abf Mon Sep 17 00:00:00 2001 From: Ivan Golikov Date: Wed, 13 Mar 2024 18:41:39 +0100 Subject: [PATCH] New docker compose configuration (#187) * Refactored Compose files * Removed unnecessary overrides * Configurable default network name * Added override for using prebuilt development version image * Tagging release candidates build with 'rc' tag * Removed service name from RC override * Updated README * Fixed condition in Actions workflow * Fixed PR CI workflow * Fixed PR CI workflow once again * Fixed condition in CI workflow Also removed RUN_ON_MASTER variable because we don't need it: on merge to master the workflow is not triggered, and on tag push the event will be 'semver', not 'branch' * Added depends_on to the api service * Fixed selenoid network name for connection * Added restart policy Stop services only after they are explicitly stopped. This enables automatic application startup after system reboot * Removed unnecessary chrome service * Using start_celery.sh script by default * Extended .dockerignore * Added external network for Selenoid to compensate removal of Chrome service * Storing Mongo data in a Docker volume instead of project directory * Fixed PR CI workflow * Should be working without new network * Legacy directory removed from .gitignore --- .dockerignore | 11 +++ .github/workflows/docker-image-update.yml | 8 +- .github/workflows/pull-request.yml | 2 +- .gitignore | 2 + Dockerfile | 6 +- README.md | 15 ++-- app/selenium_app.py | 2 +- docker-compose-rc.yaml | 63 --------------- docker-compose-stable.yaml | 73 ----------------- docker-compose.dev.yaml | 77 ------------------ docker-compose.override.dev.yaml | 18 +++++ docker-compose.override.prebuilt-dev.yaml | 6 ++ docker-compose.override.rc.yaml | 10 +++ docker-compose.remote_server.yaml | 92 ---------------------- docker-compose.remote_server_dev.yaml | 95 ----------------------- docker-compose.yaml | 69 ++++++++-------- start_celery.sh | 2 +- 17 files changed, 104 insertions(+), 447 deletions(-) delete mode 100644 docker-compose-rc.yaml delete mode 100644 docker-compose-stable.yaml delete mode 100644 docker-compose.dev.yaml create mode 100644 docker-compose.override.dev.yaml create mode 100644 docker-compose.override.prebuilt-dev.yaml create mode 100644 docker-compose.override.rc.yaml delete mode 100644 docker-compose.remote_server.yaml delete mode 100644 docker-compose.remote_server_dev.yaml diff --git a/.dockerignore b/.dockerignore index d184e5f1..b41d41a5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -36,3 +36,14 @@ data/uploaded_dataset data/other **/notebooks + +# Git data is not needed in Docker container +.git/ +.gitignore +.gitmodules + +# These scripts are unnecessary in Docker container +build_docker.sh +create-env.bat +remove-env.bat +run_docker.sh diff --git a/.github/workflows/docker-image-update.yml b/.github/workflows/docker-image-update.yml index 6f425f81..e65b87e2 100644 --- a/.github/workflows/docker-image-update.yml +++ b/.github/workflows/docker-image-update.yml @@ -9,6 +9,9 @@ on: branches: - master +env: + RUN_ON_RC: ${{ startsWith( github.head_ref, 'release/' ) }} + jobs: build-and-push: runs-on: ubuntu-latest @@ -28,10 +31,11 @@ jobs: org.opencontainers.image.title=JDI QASP ML org.opencontainers.image.description=Backend for JDN Chrome plugin tags: | - type=ref,event=branch,enable=${{ github.ref != 'refs/heads/master' }} - type=ref,event=pr + type=ref,event=branch + type=ref,event=pr,enable=${{ env.RUN_ON_RC == 'false' }} type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} + type=raw,value=rc,enable=${{ env.RUN_ON_RC == 'true' }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 560e889e..9dc33b2c 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -9,7 +9,7 @@ jobs: lint: runs-on: ubuntu-latest env: - COMPOSE_FILE: docker-compose.dev.yaml + COMPOSE_FILE: docker-compose.yaml:docker-compose.override.dev.yaml SELENOID_PARALLEL_SESSIONS_COUNT: 2 steps: - name: Checkout repository diff --git a/.gitignore b/.gitignore index 18b4031b..5555ff41 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,5 @@ coverage.xml .env .env.rc .env.dev + +docker-compose.override.yaml diff --git a/Dockerfile b/Dockerfile index 63524b72..8e898333 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,13 +11,15 @@ RUN apt install -y curl wget mc gcc make ENV APP_HOME=/jdi-qasp-ml WORKDIR ${APP_HOME} -COPY Pipfile* ${APP_HOME}/ +COPY Pipfile* ./ RUN pip install -U pip && \ pip install pipenv RUN pipenv install --ignore-pipfile --system --deploy -COPY . ${APP_HOME} +COPY . ./ + +RUN chmod +x start_celery.sh ENV PYTHONPATH=${PYTHONPATH}:/jdi-qasp-ml diff --git a/README.md b/README.md index 1e03bbf4..a694fab3 100644 --- a/README.md +++ b/README.md @@ -167,33 +167,34 @@ To validate models quality we use test web-pages, placed in directory **notebook # Docker ## Take docker image from github: -### RC version +### Release version #### macOS ```shell -export SELENOID_PARALLEL_SESSIONS_COUNT=$(sysctl -n hw.ncpu) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:rc --force && curl --output docker-compose.yaml --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/rc/docker-compose-rc.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/rc/browsers.json && docker-compose up +export SELENOID_PARALLEL_SESSIONS_COUNT=$(sysctl -n hw.ncpu) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:latest --force && curl --output docker-compose.yaml --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/master/docker-compose.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/master/browsers.json && docker-compose up ``` #### Linux ```shell -export SELENOID_PARALLEL_SESSIONS_COUNT=$(nproc) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:rc --force && curl --output docker-compose.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/rc/docker-compose-rc.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/rc/browsers.json && docker compose up +export SELENOID_PARALLEL_SESSIONS_COUNT=$(nproc) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:latest --force && curl --output docker-compose.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/master/docker-compose.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/master/browsers.json && docker compose up ``` #### Windows ```shell -set SELENOID_PARALLEL_SESSIONS_COUNT=%NUMBER_OF_PROCESSORS%&& docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:rc --force && curl.exe --output docker-compose.yaml --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/rc/docker-compose-rc.yaml && curl.exe --output browsers.json --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/rc/browsers.json && docker compose up +set SELENOID_PARALLEL_SESSIONS_COUNT=%NUMBER_OF_PROCESSORS%&& docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:latest --force && curl.exe --output docker-compose.yaml --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/master/docker-compose.yaml && curl.exe --output browsers.json --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/master/browsers.json && docker compose up ``` ### Development version #### macOS ```shell -export SELENOID_PARALLEL_SESSIONS_COUNT=$(sysctl -n hw.ncpu) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:latest --force && curl --output docker-compose.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/browsers.json && docker-compose up +export SELENOID_PARALLEL_SESSIONS_COUNT=$(sysctl -n hw.ncpu) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:develop --force && curl --output docker-compose.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.yaml && curl --output docker-compose.override.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.override.prebuilt-dev.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/browsers.json && docker-compose up ``` #### Linux ```shell -export SELENOID_PARALLEL_SESSIONS_COUNT=$(nproc) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:latest --force && curl --output docker-compose.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/browsers.json && docker compose up +export SELENOID_PARALLEL_SESSIONS_COUNT=$(nproc) && docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:develop --force && curl --output docker-compose.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.yaml && curl --output docker-compose.override.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.override.prebuilt-dev.yaml && curl --output browsers.json https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/browsers.json && docker compose up ``` #### Windows ```shell -set SELENOID_PARALLEL_SESSIONS_COUNT=%NUMBER_OF_PROCESSORS%&& docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:latest --force && curl.exe --output docker-compose.yaml --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.yaml && curl.exe --output browsers.json --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/browsers.json && docker compose up +set SELENOID_PARALLEL_SESSIONS_COUNT=%NUMBER_OF_PROCESSORS%&& docker rm --force jdi-qasp-ml-api && docker image rm ghcr.io/jdi-testing/jdi-qasp-ml:develop --force && curl.exe --output docker-compose.yaml --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.yaml && curl.exe --output docker-compose.override.yaml https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/docker-compose.override.prebuilt-dev.yaml && curl.exe --output browsers.json --url https://raw.githubusercontent.com/jdi-testing/jdi-qasp-ml/develop/browsers.json && docker compose up ``` + ## Installing version from any other repository branch: Example with branch "branch_name": diff --git a/app/selenium_app.py b/app/selenium_app.py index dfcfe73a..67f4e865 100644 --- a/app/selenium_app.py +++ b/app/selenium_app.py @@ -28,7 +28,7 @@ def get_webdriver() -> webdriver.Remote: } return webdriver.Remote( - command_executor="http://jdi-qasp-ml-selenoid:4444/wd/hub", + command_executor="http://selenoid:4444/wd/hub", desired_capabilities=capabilities, options=chrome_options, ) diff --git a/docker-compose-rc.yaml b/docker-compose-rc.yaml deleted file mode 100644 index 53aba8f2..00000000 --- a/docker-compose-rc.yaml +++ /dev/null @@ -1,63 +0,0 @@ -version: '3.9' - -name: jdi-qasp-ml -services: - chrome: - image: selenoid/vnc_chrome:118.0 - attach: false - - selenoid: - container_name: "jdi-qasp-ml-selenoid" - image: "aerokube/selenoid:latest" - ports: - - "4445:4444" - volumes: - - ".:/etc/selenoid" - - "./target:/output" - - "/var/run/docker.sock:/var/run/docker.sock" - - "./target:/opt/selenoid/video" - environment: - - "OVERRIDE_VIDEO_OUTPUT_DIR=$PWD/target" - command: - - "-conf" - - "/etc/selenoid/browsers.json" - - "-video-output-dir" - - "/opt/selenoid/video" - - "-container-network" - - "jdi-qasp-ml_default" - - "-limit" - - "${SELENOID_PARALLEL_SESSIONS_COUNT:?set it to the number of parallel running threads supported by your processor (-2 optionally)}" - - api: - image: "ghcr.io/jdi-testing/jdi-qasp-ml:rc" - container_name: jdi-qasp-ml-api - command: uvicorn app.main:api --host 0.0.0.0 --port 5000 - ports: - - "5050:5000" - logging: - driver: "json-file" - options: - max-size: "256m" - environment: - ENV: LOCAL - SELENOID_PARALLEL_SESSIONS_COUNT: $SELENOID_PARALLEL_SESSIONS_COUNT - - celery: - image: "ghcr.io/jdi-testing/jdi-qasp-ml:rc" - container_name: jdi-qasp-ml-celery - command: bash -c "chmod +x start_celery.sh && ./start_celery.sh" - logging: - driver: "json-file" - options: - max-size: "256m" - - redis: - image: "redis:alpine" - container_name: jdi-qasp-ml-redis - command: redis-server --maxmemory 2048mb --maxmemory-policy allkeys-lru - ports: - - "6379" - logging: - driver: "json-file" - options: - max-size: "256m" diff --git a/docker-compose-stable.yaml b/docker-compose-stable.yaml deleted file mode 100644 index 0c6820eb..00000000 --- a/docker-compose-stable.yaml +++ /dev/null @@ -1,73 +0,0 @@ -version: '3' - -name: jdi-qasp-ml -services: - chrome: - image: selenoid/vnc_chrome:118.0 - attach: false - - selenoid: - container_name: "jdi-qasp-ml-selenoid" - image: "aerokube/selenoid:latest" - ports: - - "4445:4444" - volumes: - - ".:/etc/selenoid" - - "./target:/output" - - "/var/run/docker.sock:/var/run/docker.sock" - - "./target:/opt/selenoid/video" - environment: - - "OVERRIDE_VIDEO_OUTPUT_DIR=$PWD/target" - command: - - "-conf" - - "/etc/selenoid/browsers.json" - - "-video-output-dir" - - "/opt/selenoid/video" - - "-container-network" - - "jdi-qasp-ml_default" - - "-limit" - - "${SELENOID_PARALLEL_SESSIONS_COUNT:?set it to the number of parallel running threads supported by your processor (-2 optionally)}" - - api: - image: "ghcr.io/jdi-testing/jdi-qasp-ml:stable" - container_name: jdi-qasp-ml-api - command: uvicorn app.main:api --host 0.0.0.0 --port 5000 - environment: - - SELENOID_PARALLEL_SESSIONS_COUNT=$SELENOID_PARALLEL_SESSIONS_COUNT - ports: - - "5050:5000" - logging: - driver: "json-file" - options: - max-size: "256m" - - celery: - image: "ghcr.io/jdi-testing/jdi-qasp-ml:stable" - container_name: jdi-qasp-ml-celery - command: celery -A app.celery_app:celery_app worker -l info - logging: - driver: "json-file" - options: - max-size: "256m" - - redis: - image: "redis:alpine" - container_name: jdi-qasp-ml-redis - command: redis-server --maxmemory 2048mb --maxmemory-policy allkeys-lru - ports: - - "6379" - logging: - driver: "json-file" - options: - max-size: "256m" - - mongodb: - image: mongo:5.0 - ports: - - 27017:27017 - volumes: - - ~/apps/mongo:/data/db - logging: - driver: "json-file" - options: - max-size: "256m" diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml deleted file mode 100644 index 48c66544..00000000 --- a/docker-compose.dev.yaml +++ /dev/null @@ -1,77 +0,0 @@ -name: jdi-qasp-ml -services: - chrome: - image: selenoid/vnc_chrome:118.0 - attach: false - - selenoid: - container_name: "jdi-qasp-ml-selenoid" - image: "aerokube/selenoid:latest" - ports: - - "4445:4444" - volumes: - - ".:/etc/selenoid" - - "./target:/output" - - "/var/run/docker.sock:/var/run/docker.sock" - - "./target:/opt/selenoid/video" - environment: - - "OVERRIDE_VIDEO_OUTPUT_DIR=$PWD/target" - command: - - "-conf" - - "/etc/selenoid/browsers.json" - - "-video-output-dir" - - "/opt/selenoid/video" - - "-container-network" - - "jdi-qasp-ml_default" - - "-limit" - - "${SELENOID_PARALLEL_SESSIONS_COUNT:?set it to the number of parallel running threads supported by your processor (-2 optionally)}" - - api: - container_name: jdi-qasp-ml-api - command: uvicorn app.main:api --host 0.0.0.0 --port 5050 - build: - context: . - dockerfile: Dockerfile - ports: - - "5050:5050" - volumes: - - .:/jdi-qasp-ml - environment: - - SELENOID_PARALLEL_SESSIONS_COUNT=$SELENOID_PARALLEL_SESSIONS_COUNT - logging: - driver: "json-file" - options: - max-size: "256m" - - celery: - container_name: jdi-qasp-ml-celery - command: celery -A app.celery_app:celery_app worker -l info - build: - context: . - dockerfile: Dockerfile - logging: - driver: "json-file" - options: - max-size: "256m" - - redis: - container_name: jdi-qasp-ml-redis - image: "redis:alpine" - command: redis-server --maxmemory 2048mb --maxmemory-policy allkeys-lru - ports: - - "6379" - logging: - driver: "json-file" - options: - max-size: "256m" - - mongodb: - image: mongo:5.0 - ports: - - 27017:27017 - volumes: - - ~/apps/mongo:/data/db - logging: - driver: "json-file" - options: - max-size: "256m" diff --git a/docker-compose.override.dev.yaml b/docker-compose.override.dev.yaml new file mode 100644 index 00000000..d8113d91 --- /dev/null +++ b/docker-compose.override.dev.yaml @@ -0,0 +1,18 @@ +services: + selenoid: + ports: + - "${SELENOID_PORT:-4445}:4444" + + api: + image: !reset null + build: + context: . + dockerfile: Dockerfile + volumes: + - .:/jdi-qasp-ml + + celery: + image: !reset null + build: + context: . + dockerfile: Dockerfile diff --git a/docker-compose.override.prebuilt-dev.yaml b/docker-compose.override.prebuilt-dev.yaml new file mode 100644 index 00000000..355a2c74 --- /dev/null +++ b/docker-compose.override.prebuilt-dev.yaml @@ -0,0 +1,6 @@ +services: + api: + image: "ghcr.io/jdi-testing/jdi-qasp-ml:develop" + + celery: + image: "ghcr.io/jdi-testing/jdi-qasp-ml:develop" diff --git a/docker-compose.override.rc.yaml b/docker-compose.override.rc.yaml new file mode 100644 index 00000000..fc575509 --- /dev/null +++ b/docker-compose.override.rc.yaml @@ -0,0 +1,10 @@ +services: + api: + image: "ghcr.io/jdi-testing/jdi-qasp-ml:rc" + environment: + ENV: LOCAL + + celery: + image: "ghcr.io/jdi-testing/jdi-qasp-ml:rc" + + mongodb: !reset [] diff --git a/docker-compose.remote_server.yaml b/docker-compose.remote_server.yaml deleted file mode 100644 index 4e856e45..00000000 --- a/docker-compose.remote_server.yaml +++ /dev/null @@ -1,92 +0,0 @@ -version: '3' - -name: jdi-qasp-ml -services: - chrome: - image: selenoid/vnc_chrome:118.0 - attach: false - - selenoid: - container_name: "jdi-qasp-ml-selenoid" - image: "aerokube/selenoid:latest" - ports: - - "4445:4444" - volumes: - - ".:/etc/selenoid" - - "./target:/output" - - "/var/run/docker.sock:/var/run/docker.sock" - - "./target:/opt/selenoid/video" - environment: - - "OVERRIDE_VIDEO_OUTPUT_DIR=$PWD/target" - command: - - "-conf" - - "/etc/selenoid/browsers.json" - - "-video-output-dir" - - "/opt/selenoid/video" - - "-container-network" - - "jdi-qasp-ml_default" - - "-limit" - - "${SELENOID_PARALLEL_SESSIONS_COUNT:?set it to the number of parallel running threads supported by your processor (-2 optionally)}" - - api: - container_name: jdi-qasp-ml-api - command: uvicorn app.main:api --host 0.0.0.0 --port 80 - build: - context: . - dockerfile: Dockerfile - ports: - - "80:80" - volumes: - - .:/jdi-qasp-ml - env_file: - - .env.rc - environment: - - SELENOID_PARALLEL_SESSIONS_COUNT=$SELENOID_PARALLEL_SESSIONS_COUNT - logging: - driver: "json-file" - options: - max-size: "256m" - - celery: - container_name: jdi-qasp-ml-celery - command: celery -A app.celery_app:celery_app worker -l info - build: - context: . - dockerfile: Dockerfile - env_file: - - .env.rc - logging: - driver: "json-file" - options: - max-size: "256m" - - redis: - container_name: jdi-qasp-ml-redis - image: "redis:alpine" - command: - - /bin/sh - - -c - # - Double dollars, so that the variable is not expanded by Docker Compose - # - Surround by quotes, so that the shell does not split the password - # - The ${variable:?message} syntax causes shell to exit with a non-zero - # code and print a message, when the variable is not set or empty - - redis-server --requirepass "$${REDIS_PASSWORD:?REDIS_PASSWORD variable is not set}" --maxmemory 2048mb --maxmemory-policy allkeys-lru - ports: - - "6379" - env_file: - - .env.rc - logging: - driver: "json-file" - options: - max-size: "256m" - - mongodb: - image: mongo:5.0 - volumes: - - ~/apps/mongo:/data/db - env_file: - - .env.rc - logging: - driver: "json-file" - options: - max-size: "256m" diff --git a/docker-compose.remote_server_dev.yaml b/docker-compose.remote_server_dev.yaml deleted file mode 100644 index 2abaf1fd..00000000 --- a/docker-compose.remote_server_dev.yaml +++ /dev/null @@ -1,95 +0,0 @@ -name: jdi-qasp-ml -services: - chrome: - image: selenoid/vnc_chrome:118.0 - attach: false - - selenoid: - container_name: "jdi-qasp-ml-selenoid" - image: "aerokube/selenoid:latest" - ports: - - "4445:4444" - volumes: - - ".:/etc/selenoid" - - "./target:/output" - - "/var/run/docker.sock:/var/run/docker.sock" - - "./target:/opt/selenoid/video" - environment: - - "OVERRIDE_VIDEO_OUTPUT_DIR=$PWD/target" - command: - - "-conf" - - "/etc/selenoid/browsers.json" - - "-video-output-dir" - - "/opt/selenoid/video" - - "-container-network" - - "${JDI_DEFAULT_NETWORK_NAME:-jdi-qasp-ml-dev-default}" - - "-limit" - - "${SELENOID_PARALLEL_SESSIONS_COUNT:?set it to the number of parallel running threads supported by your processor (-2 optionally)}" - - api-dev: - container_name: jdi-qasp-ml-api-dev - command: uvicorn app.main:api --host 0.0.0.0 --port 80 - build: - context: . - dockerfile: Dockerfile - ports: - - "5000:80" - volumes: - - .:/jdi-qasp-ml - env_file: - - .env.dev - environment: - - SELENOID_PARALLEL_SESSIONS_COUNT=$SELENOID_PARALLEL_SESSIONS_COUNT - logging: - driver: "json-file" - options: - max-size: "256m" - - celery-dev: - container_name: jdi-qasp-ml-celery-dev - command: celery -A app.celery_app:celery_app worker -l info - build: - context: . - dockerfile: Dockerfile - env_file: - - .env.dev - logging: - driver: "json-file" - options: - max-size: "256m" - - redis-dev: - container_name: jdi-qasp-ml-redis-dev - image: "redis:alpine" - command: - - /bin/sh - - -c - # - Double dollars, so that the variable is not expanded by Docker Compose - # - Surround by quotes, so that the shell does not split the password - # - The ${variable:?message} syntax causes shell to exit with a non-zero - # code and print a message, when the variable is not set or empty - - redis-server --requirepass "$${REDIS_PASSWORD:?REDIS_PASSWORD variable is not set}" --maxmemory 2048mb --maxmemory-policy allkeys-lru - ports: - - "6380" - env_file: - - .env.dev - logging: - driver: "json-file" - options: - max-size: "256m" - - mongodb-dev: - container_name: jdi-qasp-ml-mongodb-dev - image: mongo:5.0 - volumes: - - ~/apps/mongo:/data/db_dev - env_file: - - .env.dev - logging: - driver: "json-file" - options: - max-size: "256m" - -networks: - default: - name: ${JDI_DEFAULT_NETWORK_NAME:-jdi-qasp-ml-dev-default} diff --git a/docker-compose.yaml b/docker-compose.yaml index 0433b23e..5179947f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,16 +1,15 @@ -version: '3.9' +x-logging: &logging + logging: + driver: "json-file" + options: + max-size: "256m" -name: jdi-qasp-ml -services: - chrome: - image: selenoid/vnc_chrome:118.0 - attach: false +x-restart-policy: &restart + restart: unless-stopped +services: selenoid: - container_name: "jdi-qasp-ml-selenoid" image: "aerokube/selenoid:latest" - ports: - - "4445:4444" volumes: - ".:/etc/selenoid" - "./target:/output" @@ -18,46 +17,50 @@ services: - "./target:/opt/selenoid/video" environment: - "OVERRIDE_VIDEO_OUTPUT_DIR=$PWD/target" - command: + command: - "-conf" - "/etc/selenoid/browsers.json" - "-video-output-dir" - "/opt/selenoid/video" - "-container-network" - - "jdi-qasp-ml_default" + - "${JDI_DEFAULT_NETWORK_NAME:-jdi-qasp-ml-default}" - "-limit" - "${SELENOID_PARALLEL_SESSIONS_COUNT:?set it to the number of parallel running threads supported by your processor (-2 optionally)}" + <<: *restart api: image: "ghcr.io/jdi-testing/jdi-qasp-ml:latest" - container_name: jdi-qasp-ml-api - command: uvicorn app.main:api --host 0.0.0.0 --port 5000 + command: uvicorn app.main:api --host 0.0.0.0 --port 5050 + depends_on: + - selenoid + - celery + - redis + - mongodb ports: - - "5050:5000" - logging: - driver: "json-file" - options: - max-size: "256m" + - "${APP_EXPOSE_PORT:-5050}:5050" environment: - ENV: LOCAL - SELENOID_PARALLEL_SESSIONS_COUNT: $SELENOID_PARALLEL_SESSIONS_COUNT + - SELENOID_PARALLEL_SESSIONS_COUNT=$SELENOID_PARALLEL_SESSIONS_COUNT + <<: [*logging, *restart] celery: image: "ghcr.io/jdi-testing/jdi-qasp-ml:latest" - container_name: jdi-qasp-ml-celery - command: bash -c "chmod +x start_celery.sh && ./start_celery.sh" - logging: - driver: "json-file" - options: - max-size: "256m" + command: ./start_celery.sh + <<: [*logging, *restart] redis: image: "redis:alpine" - container_name: jdi-qasp-ml-redis command: redis-server --maxmemory 2048mb --maxmemory-policy allkeys-lru - ports: - - "6379" - logging: - driver: "json-file" - options: - max-size: "256m" + <<: [*logging, *restart] + + mongodb: + image: mongo:5.0 + volumes: + - mongo-data:/data/db + <<: [*logging, *restart] + +networks: + default: + name: ${JDI_DEFAULT_NETWORK_NAME:-jdi-qasp-ml-default} + +volumes: + mongo-data: diff --git a/start_celery.sh b/start_celery.sh index 8d17508f..cd7c5aa0 100644 --- a/start_celery.sh +++ b/start_celery.sh @@ -1,5 +1,5 @@ #!/bin/bash CORES_TO_USE=$(($(nproc)-2>0?$(nproc)-2:1)) -echo $CORES_TO_USE +echo "Celery will use $CORES_TO_USE cores" celery -A app.celery_app:celery_app worker -l info --concurrency $CORES_TO_USE