TEST-modin-project#2008: Reduce runtime of CI checks a lot (modin-project#6356)

Signed-off-by: Vasily Litvinov <fam1ly.n4me@yandex.ru>

Author: vnlitvinov

.github/workflows/ci-notebooks.yml

@@ -7,6 +7,7 @@ on:
       - .github/workflows/ci-notebooks.yml
       - setup.cfg
       - setup.py
+      - requirements/env_hdk.yml
 concurrency:
   # Cancel other jobs in the same branch. We don't care whether CI passes
   # on old commits.
@@ -26,41 +27,19 @@ jobs:
         execution: [pandas_on_ray, pandas_on_dask, pandas_on_unidist, hdk_on_native]
     steps:
       - uses: actions/checkout@v3
-        with:
-          fetch-depth: 1
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.8.x"
-          architecture: "x64"
+      - uses: ./.github/actions/python-only
         if: matrix.execution != 'hdk_on_native'
-      - name: Cache conda
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/conda_pkgs_dir
-            ~/.cache/pip
-          key:
-            ${{ runner.os }}-conda-${{ hashFiles('requirements/env_hdk.yml') }}
-        if: matrix.execution == 'hdk_on_native'
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: ./.github/actions/mamba-env
         with:
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
-          use-mamba: true
-          activate-environment: modin_on_hdk
           environment-file: requirements/env_hdk.yml
-          python-version: 3.8
-          channel-priority: strict
-          # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed
-          # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264
-          use-only-tar-bz2: false
+          activate-environment: modin_on_hdk
         if: matrix.execution == 'hdk_on_native'
       - name: Cache datasets
         uses: actions/cache@v2
         with:
           path: taxi.csv
           # update cache only if notebooks require it to be changed
-          key: hashFiles("examples/tutorial/jupyter/**")
+          key: taxi-csv-dataset-${{ hashFiles('examples/tutorial/jupyter/**') }}
       # replace modin with . in the tutorial requirements file for `pandas_on_ray` and
       # `pandas_on_dask` since we need Modin built from sources
       - run: sed -i 's/modin/./g' examples/tutorial/jupyter/execution/${{ matrix.execution }}/requirements.txt

.github/workflows/ci-required.yml

@@ -5,7 +5,11 @@ concurrency:
   # on old commits.
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+env:
+  MODIN_GITHUB_CI: true
+
 jobs:
+
   check-pr-title:
     runs-on: ubuntu-latest
     steps:
@@ -14,6 +18,7 @@ jobs:
         # NOTE: If you change the allowed prefixes here, update
         # the documentation about them in /docs/development/contributing.rst
         regexp: '^(?:FEAT|DOCS|FIX|REFACTOR|TEST|PERF)-#\d+:'
+
   build-docs:
     name: build docs
     runs-on: ubuntu-latest
@@ -29,5 +34,77 @@ jobs:
           cache-dependency-path: '**/requirements-doc.txt'
       - run: pip install -r docs/requirements-doc.txt
       - run: cd docs && sphinx-build -T -E -W -b html . build
-env:
-  MODIN_GITHUB_CI: true
+
+  lint-pydocstyle:
+    name: lint (pydocstyle)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/python-only
+      # The `numpydoc` version here MUST match the versions in the dev requirements files.
+      - run: pip install pytest pytest-cov pydocstyle numpydoc==1.1.0 xgboost
+      - run: python -m pytest scripts/test
+      - run: pip install -e ".[all]"
+      - run: |
+          python scripts/doc_checker.py --add-ignore=D101,D102,D103,D105 --disable-numpydoc \
+            modin/pandas/dataframe.py modin/pandas/series.py \
+            modin/pandas/groupby.py \
+            modin/pandas/series_utils.py modin/pandas/general.py \
+            modin/pandas/plotting.py modin/pandas/utils.py \
+            modin/pandas/iterator.py modin/pandas/indexing.py \
+      - run: python scripts/doc_checker.py modin/core/dataframe
+      - run: python scripts/doc_checker.py modin/core/execution/dask
+      - run: |
+          python scripts/doc_checker.py \
+            modin/pandas/accessor.py modin/pandas/general.py \
+            modin/pandas/groupby.py modin/pandas/indexing.py \
+            modin/pandas/iterator.py modin/pandas/plotting.py \
+            modin/pandas/series_utils.py modin/pandas/utils.py \
+            modin/pandas/base.py \
+            modin/pandas/io.py \
+            asv_bench/benchmarks/utils \
+            asv_bench/benchmarks/__init__.py asv_bench/benchmarks/io/__init__.py \
+            asv_bench/benchmarks/scalability/__init__.py \
+            modin/core/io \
+            modin/experimental/core/execution/ray/implementations/pandas_on_ray \
+            modin/experimental/core/execution/ray/implementations/pyarrow_on_ray \
+            modin/pandas/series.py \
+            modin/core/execution/python \
+            modin/pandas/dataframe.py \
+            modin/config/__init__.py \
+            modin/config/__main__.py \
+            modin/config/envvars.py \
+            modin/config/pubsub.py
+      - run: python scripts/doc_checker.py modin/distributed
+      - run: python scripts/doc_checker.py modin/utils.py
+      - run: python scripts/doc_checker.py modin/experimental/sklearn
+      - run: |
+          python scripts/doc_checker.py modin/experimental/xgboost/__init__.py \
+            modin/experimental/xgboost/utils.py modin/experimental/xgboost/xgboost.py \
+            modin/experimental/xgboost/xgboost_ray.py
+      - run: python scripts/doc_checker.py modin/core/execution/ray
+      - run: |
+          python scripts/doc_checker.py modin/core/execution/dispatching/factories/factories.py \
+            modin/core/execution/dispatching/factories/dispatcher.py                            \
+      - run: python scripts/doc_checker.py scripts/doc_checker.py
+      - run: |
+          python scripts/doc_checker.py modin/experimental/pandas/io.py \
+            modin/experimental/pandas/numpy_wrap.py modin/experimental/pandas/__init__.py
+      - run: python scripts/doc_checker.py modin/core/storage_formats/base
+      - run: python scripts/doc_checker.py modin/experimental/core/storage_formats/pyarrow
+      - run: python scripts/doc_checker.py modin/core/storage_formats/pandas
+      - run: |
+          python scripts/doc_checker.py \
+            modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe \
+            modin/experimental/core/execution/native/implementations/hdk_on_native/io \
+            modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning \
+            modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_algebra.py \
+            modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_builder.py \
+            modin/experimental/core/execution/native/implementations/hdk_on_native/calcite_serializer.py \
+            modin/experimental/core/execution/native/implementations/hdk_on_native/df_algebra.py \
+            modin/experimental/core/execution/native/implementations/hdk_on_native/expr.py \
+            modin/experimental/core/execution/native/implementations/hdk_on_native/hdk_worker.py \
+      - run: python scripts/doc_checker.py modin/experimental/core/storage_formats/hdk
+      - run: python scripts/doc_checker.py modin/experimental/core/execution/native/implementations/hdk_on_native/interchange/dataframe_protocol
+      - run: python scripts/doc_checker.py modin/experimental/batch/pipeline.py
+      - run: python scripts/doc_checker.py modin/logging