Housekeeping (DistrictDataLabs#1187)

This PR fixes DistrictDataLabs#1113 by handling some housekeeping issues. We've changed our master branch to the main branch and created a release issue template (though I don't think it is used automatically by GitHub). We've also checked the new pip resolver and all of our dependencies are compatible. This PR also updates our CI to test Python 3.8 and 3.9 to ensure we're testing on the latest version of Python.

Author: bbengfort

.gitattributes

@@ -0,0 +1,20 @@
+# Set the default behavior, in case contributors don't have core.autocrlf set.
+* text=auto
+
+# Explicitly declare text files
+*.py text
+*.rst text
+*.md text
+*.json text
+*.ipynb text
+*.cfg text
+
+# Baseline images are binary and should not be modified
+*.png binary
+*.jpg binary
+*.pdf binary
+
+# Compressed files are binary and should not be modified
+*.gz binary
+*.npz binary
+*.zip binary

.github/RELEASE_TEMPLATE.md

@@ -0,0 +1,14 @@
+**Deployed:** DayOfWeek, Month D, YYYY
+**Current Contributors:** [insert GitHub @username comma list]
+
+[Release description (1-2 paras)]
+
+**Major Changes:**
+
+- [Change 1]
+- [Change 2]
+
+Minor Changes:
+
+- [Change 1]
+- [Change 2]

.github/workflows/ci.yml

@@ -3,104 +3,128 @@
 
 name: Yellowbrick CI
 
-on: 
-  # Trigger on push to the primary branches only
+on:
+  # Trigger on push to the primary branches and all versioned tags
   push:
     branches:
       - develop
       - main
-      - master
+    tags:
+      - 'v*'
   # Trigger on pull request always (note the trailing colon)
   pull_request:
 
 jobs:
+  # Test Vanilla Python (using PyPI for dependencies) as a matrix.
+  test_pypi:
+    runs-on: ${{ matrix.os }}
+    name: PyPi (${{ matrix.python-version }}, ${{ matrix.os }})
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.8, 3.9]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r tests/requirements.txt
+        python -m nltk.downloader popular
+
+    - name: Run Unit Tests
+      run: |
+        make test
+
+    - name: Upload Coverage
+      uses: codecov/codecov-action@v1
+      with:
+        env_vars: OS,PYTHON
+        name: codecov-umbrella
+        fail_ci_if_error: false
+
+  # Test Anaconda Python (using conda-forge for dependencies)
   test_conda:
-    name: Conda (${{ matrix.python-version }}, ${{ matrix.os }})
     runs-on: ${{ matrix.os }}
+    name: Conda (${{ matrix.python-version }}, ${{ matrix.os }})
     strategy:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
-        python-version: ["3.7", "3.8"]
+        python-version: ["3.8", "3.9"]
+
     steps:
-      - uses: actions/checkout@v2
-      - uses: conda-incubator/setup-miniconda@v2
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Setup Conda ${{ matrix.python-version }}
+        uses: conda-incubator/setup-miniconda@v2
         with:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}
           channels: conda-forge,spyder-ide
           channel-priority: flexible
 
-      - name: Conda info
+      - name: Conda Info
         shell: bash -l {0}
         run: conda info
-      - name: Add conda to system path
+
+      - name: Add conda to System $PATH
         shell: bash -l {0}
         run: |
           # $CONDA is an environment variable pointing to the root of the miniconda directory
           echo $CONDA/bin >> $GITHUB_PATH
-      - name: Install dependencies
+
+      - name: Install Dependencies
         shell: bash -l {0}
         run: |
           conda config --set always_yes yes --set changeps1 no
           conda update -n base conda --yes
           conda env create -f tests/requirements.txt -n yellowbrick python=${{ matrix.python-version }}
-      - name: Test with pytest
+          conda activate yellowbrick
+          python -m nltk.downloader popular
+
+      - name: Run Unit Tests
         shell: bash -l {0}
         run: |
           conda activate yellowbrick
-          python -m nltk.downloader popular
           make test
 
-      - name: Upload coverage to Codecov
+      - name: Upload Coverage
         uses: codecov/codecov-action@v1
         with:
           env_vars: OS,PYTHON
           name: codecov-umbrella
           fail_ci_if_error: false
 
-
-  test_pypi:
-    name: PyPi (${{ matrix.python-version }}, ${{ matrix.os }})
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: [3.7, 3.8]
-        os: [ubuntu-latest, macos-latest, windows-latest]
-
-    runs-on: ${{ matrix.os }}
-
+  # Test building docs using Sphinx since this also runs a lot of Yellowbrick code
+  test_docs:
+    runs-on: ubuntu-latest
+    name: Build Docs
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r tests/requirements.txt
-    - name: Test with pytest
-      run: |
-        python -m nltk.downloader popular
-        make test
-    - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v1
-      with:
-        env_vars: OS,PYTHON
-        name: codecov-umbrella
-        fail_ci_if_error: false
+      - name: Checkout Code
+        uses: actions/checkout@v2
 
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -r docs/requirements.txt
 
-#  slackNotification:
-#    name: Slack Notification
-#    needs: [test_pypi, test_conda]
-#    runs-on: ubuntu-latest
-#    steps:
-#    - uses: actions/checkout@v2
-#    - name: Slack Notification
-#      uses: rtCamp/action-slack-notify@v2
-#      env:
-#        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-#        SLACK_CHANNEL: yb-ci
-#        SLACK_USERNAME: "Github Actions"
+      - name: Run Sphinx
+        uses: ammaraskar/sphinx-action@master
+        with:
+          docs-folder: "docs/"

docs/api/model_selection/cross_validation.rst

@@ -41,7 +41,7 @@ In the following example, we show how to visualize cross-validated scores for a
     X, y = load_occupancy()
 
     # Create a cross-validation strategy
-    cv = StratifiedKFold(n_splits=12, random_state=42)
+    cv = StratifiedKFold(n_splits=12, shuffle=True, random_state=42)
 
     # Instantiate the classification model and visualizer
     model = MultinomialNB()
@@ -72,7 +72,7 @@ In this next example we show how to visualize cross-validated scores for a regre
     X, y = load_energy()
 
     # Instantiate the regression model and visualizer
-    cv = KFold(n_splits=12, random_state=42)
+    cv = KFold(n_splits=12, shuffle=True, random_state=42)
 
     model = Ridge()
     visualizer = CVScores(model, cv=cv, scoring='r2')
@@ -102,7 +102,7 @@ Quick Method
     X, y = load_energy()
 
     # Instantiate the regression model and visualizer
-    cv = KFold(n_splits=12, random_state=42)
+    cv = KFold(n_splits=12, shuffle=True, random_state=42)
 
     model = Ridge()
     visualizer = cv_scores(model, X, y, cv=cv, scoring='r2')

docs/conf.py

@@ -232,7 +232,7 @@
 
 
 def setup(app):
-    app.add_stylesheet("theme_overrides.css")
+    app.add_css_file("theme_overrides.css")
 
 
 # Add any extra paths that contain custom files (such as robots.txt or

docs/contributing/developing_visualizers.rst

@@ -139,12 +139,14 @@ Visual tests are notoriously difficult to create --- how do you test a visualiza
             try:
                 visualizer = MyVisualizer()
                 assert visualizer.fit(X, y) is visualizer, "fit should return self"
-                visualizer.show()
+                visualizer.finalize()
             except Exception as e:
                 pytest.fail("my visualizer didn't work: {}".format(e))
 
 This simple test case is an excellent start to a larger test package and we recommend starting with this test as you develop your visualizer. Once you've completed the development and prototyping you can start to include :ref:`test fixtures <fixtures>` and test various normal use cases and edge cases with unit tests, then build :ref:`image similarity tests <assert_images_similar>` to more thoroughly define the integration tests.
 
+.. note:: In tests you should not call ``visualizer.show()`` because this will call ``plt.show()`` and trigger a matplotlib warning that the visualization cannot be displayed using the test backend ``Agg``. Calling ``visualizer.finalize()`` instead should produce the full image and make the tests faster and more readable.
+
 
 Running the Test Suite
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -157,9 +159,15 @@ The required dependencies for the test suite include testing utilities and libra
 
 Tests can be run as follows from the project root::
 
-    $ make test
+    $ pytest
+
+The ``pytest`` function is configured via ``setup.cfg`` with the correct arguments and runner, and therefore must be run from the project root. You can also use ``make test`` but this simply runs the ``pytest`` command.
+
+The tests do take a while to run, so during normal development it is recommended that you only run the tests for the test file you're writing::
+
+    $ pytest tests/test_package/test_module.py
 
-The Makefile uses the pytest runner and testing suite as well as the coverage library.
+This will greatly simplify development and allow you to focus on the changes that you're making!
 
 .. _assert_images_similar:
 
@@ -181,7 +189,7 @@ For example, create your test function located in ``tests/test_regressor/test_my
         def test_my_visualizer_output(self):
             visualizer = MyVisualizer()
             visualizer.fit(X)
-            visualizer.show()
+            visualizer.finalize()
             self.assert_images_similar(visualizer)
 
 The first time this test is run, there will be no baseline image to compare against, so the test will fail. Copy the output images (in this case ``tests/actual_images/test_regressor/test_myvisualizer/test_my_visualizer_output.png``) to the correct subdirectory of baseline_images tree in the source directory (in this case ``tests/baseline_images/test_regressor/test_myvisualizer/test_my_visualizer_output.png``). Put this new file under source code revision control (with git add). When rerunning the tests, they should now pass.
@@ -196,6 +204,20 @@ This will move all related test images from ``actual_images`` to ``baseline_imag
 
 This is useful particularly if you're stuck trying to get an image comparison to work. For more information on the images helper script, use ``python -m tests.images --help``.
 
+Finally, to reiterate a note from above; make sure that you do not call your visualizer's ``show()`` method, instead using ``finalize()`` directly. If you call ``show()``, it will in turn call ``plt.show()`` which will issue a warning due to the fact that the tests use the ``Agg`` backend. When testing quick methods you should pass the ``show=False`` argument to the quick method as follows:
+
+.. code:: python
+
+    from tests.base import VisualTestCase
+
+    class MyVisualizerTests(VisualTestCase):
+
+        def test_my_visualizer_quickmethod(self):
+            visualizer = my_visualizer_quickmethod(X, show=False)
+            self.assert_images_similar(visualizer)
+
+Generally speaking, testing quick methods is identical to testing Visualizers except for the method of interaction because the quick method will return the visualizer or the axes object.
+
 .. _fixtures:
 
 Test Fixtures

setup.py

@@ -64,6 +64,7 @@
     "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
     "Topic :: Software Development",
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Topic :: Scientific/Engineering :: Visualization",

tests/base.py

@@ -164,10 +164,13 @@ class ImageComparison(object):
 
     visualizer : Yellowbrick Visualizer instance, optional
         An instantiated Yellowbrick visualizer that wraps a matplotlib Axes
-        and has been drawn on via the Yellowbrick API.
+        and has been drawn on via the Yellowbrick API. If a visualizer is specified,
+        generally speaking it should have a call to ``finalize()`` performed, but not a
+        call to ``show()``.
 
     ax : matplotlib Axes, optional
-        A direct reference to a matplotlib Axes that has been drawn on.
+        A direct reference to a matplotlib Axes that has been drawn on. If using the
+        Axes object, you should not call ``plt.show()``.
 
     tol : float, default: 0.01
         The tolerance as a color value difference, where 255 is the maximal

tests/test_classifier/test_prcurve.py

@@ -435,7 +435,10 @@ def test_quick_method_with_test_set(self):
         )
 
         viz = precision_recall_curve(
-            RandomForestClassifier(random_state=72), X_train, y_train, X_test, y_test
+            RandomForestClassifier(random_state=72),
+            X_train, y_train,
+            X_test, y_test,
+            show=False
         )
         self.assert_images_similar(viz)
 
@@ -453,11 +456,13 @@ def test_missing_test_data_in_quick_method(self):
 
         with pytest.raises(YellowbrickValueError, match=emsg):
             precision_recall_curve(
-                RandomForestClassifier(), X_train, y_train, y_test=y_test
+                RandomForestClassifier(), X_train, y_train, y_test=y_test, show=False
             )
 
         with pytest.raises(YellowbrickValueError, match=emsg):
-            precision_recall_curve(RandomForestClassifier(), X_train, y_train, X_test)
+            precision_recall_curve(
+                RandomForestClassifier(), X_train, y_train, X_test, show=False
+            )
 
     def test_per_class_and_micro(self):
         """
@@ -469,5 +474,5 @@ def test_per_class_and_micro(self):
         )
         with pytest.warns(YellowbrickWarning, match=msg):
             PrecisionRecallCurve(
-                RidgeClassifier(random_state=13), micro=True, per_class=True
+                RidgeClassifier(random_state=13), micro=True, per_class=True, show=False
             )