matplotlib · tacaswell · Aug 23, 2016 · Jul 12, 2016 · Jul 10, 2016 · Jul 23, 2016
diff --git a/.travis.yml b/.travis.yml
@@ -45,8 +45,13 @@ env:
     - NPROC=2
     - TEST_ARGS=--no-pep8
     - NOSE_ARGS="--processes=$NPROC --process-timeout=300"
+    - PYTEST_ARGS="-ra --timeout=300 --durations=25 --cov-report= --cov=lib" # -n $NPROC
     - PYTHON_ARGS=
     - DELETE_FONT_CACHE=
+    - USE_PYTEST=false
+    #- PYTHONHASHSEED=0  # Workaround for pytest-xdist flaky colletion order
+    #                    # https://github.com/pytest-dev/pytest/issues/920
+    #                    # https://github.com/pytest-dev/pytest/issues/1075
 
 matrix:
   include:
@@ -60,6 +65,8 @@ matrix:
       env: TEST_ARGS=--pep8
     - python: 3.5
       env: BUILD_DOCS=true
+    - python: 3.5
+      env: USE_PYTEST=true PANDAS=pandas DELETE_FONT_CACHE=1 TEST_ARGS=
     - python: "nightly"
       env: PRE=--pre
     - os: osx
@@ -107,10 +114,14 @@ install:
     # Install dependencies from pypi
     pip install $PRE python-dateutil $NUMPY pyparsing!=2.1.6 $PANDAS pep8 cycler coveralls coverage
     pip install $PRE pillow sphinx!=1.3.0 $MOCK numpydoc ipython colorspacious
+
     # Install nose from a build which has partial
     # support for python36 and suport for coverage output suppressing
     pip install git+https://github.com/jenshnielsen/nose.git@matplotlibnose
 
+    # pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124
+    pip install $PRE pytest 'pytest-cov>=2.3.1' pytest-timeout pytest-xdist pytest-faulthandler
+
     # We manually install humor sans using the package from Ubuntu 14.10. Unfortunatly humor sans is not
     # availible in the Ubuntu version used by Travis but we can manually install the deb from a later
     # version since is it basically just a .ttf file
@@ -147,16 +158,21 @@ script:
   - |
     echo Testing import of tkagg backend
     MPLBACKEND="tkagg" python -c 'import matplotlib.pyplot as plt; print(plt.get_backend())'
-    echo The following args are passed to nose $NOSE_ARGS
     if [[ $BUILD_DOCS == false ]]; then
       if [[ $DELETE_FONT_CACHE == 1 ]]; then
         rm -rf ~/.cache/matplotlib
       fi
       export MPL_REPO_DIR=$PWD  # needed for pep8-conformance test of the examples
-      if [[ $TRAVIS_OS_NAME == 'osx' ]]; then
-        python tests.py $NOSE_ARGS $TEST_ARGS
+      if [[ $USE_PYTEST == false ]]; then
+        echo The following args are passed to nose $NOSE_ARGS
+        if [[ $TRAVIS_OS_NAME == 'osx' ]]; then
+          python tests.py $NOSE_ARGS $TEST_ARGS
+        else
+          gdb -return-child-result -batch -ex r -ex bt --args python $PYTHON_ARGS tests.py $NOSE_ARGS $TEST_ARGS
+        fi
       else
-        gdb -return-child-result -batch -ex r -ex bt --args python $PYTHON_ARGS tests.py $NOSE_ARGS $TEST_ARGS
+        echo The following args are passed to pytest $PYTEST_ARGS
+        py.test $PYTEST_ARGS $TEST_ARGS
       fi
     else
       cd doc
@@ -171,6 +187,9 @@ script:
       pip install $PRE requests==2.9.2 linkchecker
       linkchecker build/html/index.html
     fi
+  # Currently disabled because of differece in behaviour
+  # between `pytest-cov` and `nose-coverage`
+  #- if [[ $USE_PYTEST == true ]]; then coveralls; fi
   - rm -rf $HOME/.cache/matplotlib/tex.cache
   - rm -rf $HOME/.cache/matplotlib/test_cache
 

diff --git a/appveyor.yml b/appveyor.yml
@@ -14,6 +14,12 @@ environment:
     CMD_IN_ENV: "cmd /E:ON /V:ON /C obvci_appveyor_python_build_env.cmd"
     # Workaround for https://github.com/conda/conda-build/issues/636
     PYTHONIOENCODING: "UTF-8"
+    TEST_ARGS: --no-pep8
+    PYTEST_ARGS: -ra --timeout=300 --durations=25 #--cov-report= --cov=lib #-n %NUMBER_OF_PROCESSORS%
+    USE_PYTEST: no
+    #PYTHONHASHSEED: 0  # Workaround for pytest-xdist flaky colletion order
+    #                   # https://github.com/pytest-dev/pytest/issues/920
+    #                   # https://github.com/pytest-dev/pytest/issues/1075
 
   matrix:
     # for testing purpose: numpy 1.8 on py2.7, for the rest use 1.10/latest
@@ -38,6 +44,13 @@ environment:
       PYTHON_VERSION: "3.5"
       TEST_ALL: "no"
       CONDA_INSTALL_LOCN: "C:\\Miniconda35-x64"
+    - TARGET_ARCH: "x64"
+      CONDA_PY: "35"
+      CONDA_NPY: "110"
+      PYTHON_VERSION: "3.5"
+      TEST_ALL: "no"
+      CONDA_INSTALL_LOCN: "C:\\Miniconda35-x64"
+      USE_PYTEST: yes
     - TARGET_ARCH: "x86"
       CONDA_PY: "27"
       CONDA_NPY: "18"
@@ -58,7 +71,7 @@ platform:
 build: false
 
 init:
-  - cmd: "ECHO %PYTHON_VERSION% %CONDA_INSTALL_LOCN%"
+  - cmd: "ECHO %PYTHON_VERSION% PYTEST=%USE_PYTEST% %CONDA_INSTALL_LOCN%"
 
 install:
   - cmd: set PATH=%CONDA_INSTALL_LOCN%;%CONDA_INSTALL_LOCN%\scripts;%PATH%;
@@ -82,10 +95,15 @@ install:
   # same things as the requirements in ci/conda_recipe/meta.yaml
   # if conda-forge gets a new pyqt, it might be nice to install it as well to have more backends
   # https://github.com/conda-forge/conda-forge.github.io/issues/157#issuecomment-223536381
-  - cmd: conda create -q -n test-environment python=%PYTHON_VERSION% pip setuptools numpy  python-dateutil freetype=2.6 msinttypes "tk=8.5" pyparsing pytz tornado "libpng>=1.6.21,<1.7" "zlib=1.2" "cycler>=0.10" nose mock
+  - conda create -q -n test-environment python=%PYTHON_VERSION%
+    pip setuptools numpy python-dateutil freetype=2.6 msinttypes "tk=8.5"
+    pyparsing pytz tornado "libpng>=1.6.21,<1.7" "zlib=1.2" "cycler>=0.10"
+    nose mock sphinx
   - activate test-environment
   - cmd: echo %PYTHON_VERSION% %TARGET_ARCH%
   - cmd: IF %PYTHON_VERSION% == 2.7 conda install -q functools32
+  # pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124
+  - if x%USE_PYTEST% == xyes conda install -q pytest "pytest-cov>=2.3.1" pytest-timeout #pytest-xdist
 
   # Let the install prefer the static builds of the libs
   - set LIBRARY_LIB=%CONDA_PREFIX%\Library\lib
@@ -124,7 +142,9 @@ test_script:
   # Test import of tkagg backend
   - python -c "import matplotlib as m; m.use('tkagg'); import matplotlib.pyplot as plt; print(plt.get_backend())"
   # tests
-  - python tests.py
+  - if x%USE_PYTEST% == xyes echo The following args are passed to pytest %PYTEST_ARGS%
+  - if x%USE_PYTEST% == xyes py.test %PYTEST_ARGS% %TEST_ARGS%
+  - if x%USE_PYTEST% == xno python tests.py %TEST_ARGS%
   # Generate a html for visual tests
   - python visual_tests.py
 

diff --git a/conftest.py b/conftest.py
@@ -0,0 +1,112 @@
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+import inspect
+import os
+import pytest
+import unittest
+
+import matplotlib
+matplotlib.use('agg')
+
+from matplotlib import default_test_modules
+from matplotlib.testing.decorators import ImageComparisonTest
+
+
+IGNORED_TESTS = {
+    'matplotlib': [
+        'test_usetex',
+    ],
+}
+
+
+def blacklist_check(path):
+    """Check if test is blacklisted and should be ignored"""
+    head, tests_dir = os.path.split(path.dirname)
+    if tests_dir != 'tests':
+        return True
+    head, top_module = os.path.split(head)
+    return path.purebasename in IGNORED_TESTS.get(top_module, [])
+
+
+def whitelist_check(path):
+    """Check if test is not whitelisted and should be ignored"""
+    left = path.dirname
+    last_left = None
+    module_path = path.purebasename
+    while len(left) and left != last_left:
+        last_left = left
+        left, tail = os.path.split(left)
+        module_path = '.'.join([tail, module_path])
+        if module_path in default_test_modules:
+            return False
+    return True
+
+
+COLLECT_FILTERS = {
+    'none': lambda _: False,
+    'blacklist': blacklist_check,
+    'whitelist': whitelist_check,
+}
+
+
+def is_nose_class(cls):
+    """Check if supplied class looks like Nose testcase"""
+    return any(name in ['setUp', 'tearDown']
+               for name, _ in inspect.getmembers(cls))
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("matplotlib", "matplotlib custom options")
+
+    group.addoption('--collect-filter', action='store',
+                    choices=COLLECT_FILTERS, default='blacklist',
+                    help='filter tests during collection phase')
+
+    group.addoption('--no-pep8', action='store_true',
+                    help='skip PEP8 compliance tests')
+
+
+def pytest_configure(config):
+    matplotlib._called_from_pytest = True
+    matplotlib._init_tests()
+
+    if config.getoption('--no-pep8'):
+        default_test_modules.remove('matplotlib.tests.test_coding_standards')
+        IGNORED_TESTS['matplotlib'] += 'test_coding_standards'
+
+
+def pytest_unconfigure(config):
+    matplotlib._called_from_pytest = False
+
+
+def pytest_ignore_collect(path, config):
+    if path.ext == '.py':
+        collect_filter = config.getoption('--collect-filter')
+        return COLLECT_FILTERS[collect_filter](path)
+
+
+def pytest_pycollect_makeitem(collector, name, obj):
+    if inspect.isclass(obj):
+        if issubclass(obj, ImageComparisonTest):
+            # Workaround `image_compare` decorator as it returns class
+            # instead of function and this confuses pytest because it crawls
+            # original names and sees 'test_*', but not 'Test*' in that case
+            return pytest.Class(name, parent=collector)
+
+        if is_nose_class(obj) and not issubclass(obj, unittest.TestCase):
+            # Workaround unittest-like setup/teardown names in pure classes
+            setup = getattr(obj, 'setUp', None)
+            if setup is not None:
+                obj.setup_method = lambda self, _: obj.setUp(self)
+            tearDown = getattr(obj, 'tearDown', None)
+            if tearDown is not None:
+                obj.teardown_method = lambda self, _: obj.tearDown(self)
+            setUpClass = getattr(obj, 'setUpClass', None)
+            if setUpClass is not None:
+                obj.setup_class = obj.setUpClass
+            tearDownClass = getattr(obj, 'tearDownClass', None)
+            if tearDownClass is not None:
+                obj.teardown_class = obj.tearDownClass
+
+            return pytest.Class(name, parent=collector)
diff --git a/lib/matplotlib/testing/__init__.py b/lib/matplotlib/testing/__init__.py
@@ -5,6 +5,7 @@
 import warnings
 from contextlib import contextmanager
 
+import matplotlib
 from matplotlib.cbook import is_string_like, iterable
 from matplotlib import rcParams, rcdefaults, use
 
@@ -14,16 +15,31 @@ def _is_list_like(obj):
     return not is_string_like(obj) and iterable(obj)
 
 
+def is_called_from_pytest():
+    """Returns whether the call was done from pytest"""
+    return getattr(matplotlib, '_called_from_pytest', False)
+
+
 def xfail(msg=""):
     """Explicitly fail an currently-executing test with the given message."""
-    from .nose import knownfail
-    knownfail(msg)
+    __tracebackhide__ = True
+    if is_called_from_pytest():
+        import pytest
+        pytest.xfail(msg)
+    else:
+        from .nose import knownfail
+        knownfail(msg)
 
 
 def skip(msg=""):
     """Skip an executing test with the given message."""
-    from nose import SkipTest
-    raise SkipTest(msg)
+    __tracebackhide__ = True
+    if is_called_from_pytest():
+        import pytest
+        pytest.skip(msg)
+    else:
+        from nose import SkipTest
+        raise SkipTest(msg)
 
 
 # stolen from pytest

diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py
@@ -27,7 +27,7 @@
 from matplotlib import rcParams
 from matplotlib.testing.compare import comparable_formats, compare_images, \
      make_test_filename
-from . import copy_metadata, skip, xfail
+from . import copy_metadata, is_called_from_pytest, skip, xfail
 from .exceptions import ImageComparisonFailure
 
 
@@ -37,8 +37,12 @@ def skipif(condition, *args, **kwargs):
 
     Optionally specify a reason for better reporting.
     """
-    from .nose.decorators import skipif
-    return skipif(condition, *args, **kwargs)
+    if is_called_from_pytest():
+        import pytest
+        return pytest.mark.skipif(condition, *args, **kwargs)
+    else:
+        from .nose.decorators import skipif
+        return skipif(condition, *args, **kwargs)
 
 
 def knownfailureif(fail_condition, msg=None, known_exception_class=None):
@@ -53,8 +57,14 @@ def knownfailureif(fail_condition, msg=None, known_exception_class=None):
     if the exception is an instance of this class. (Default = None)
 
     """
-    from .nose.decorators import knownfailureif
-    return knownfailureif(fail_condition, msg, known_exception_class)
+    if is_called_from_pytest():
+        import pytest
+        strict = fail_condition and fail_condition != 'indeterminate'
+        return pytest.mark.xfail(condition=fail_condition, reason=msg,
+                                 raises=known_exception_class, strict=strict)
+    else:
+        from .nose.decorators import knownfailureif
+        return knownfailureif(fail_condition, msg, known_exception_class)
 
 
 def _do_cleanup(original_units_registry, original_settings):
@@ -198,7 +208,7 @@ def remove_text(figure):
     def test(self):
         baseline_dir, result_dir = _image_directories(self._func)
         if self._style != 'classic':
-            xfail('temporarily disabled until 2.0 tag')
+            skip('temporarily disabled until 2.0 tag')
         for fignum, baseline in zip(plt.get_fignums(), self._baseline_images):
             for extension in self._extensions:
                 will_fail = not extension in comparable_formats()
@@ -228,7 +238,7 @@ def test(self):
                 @knownfailureif(
                     will_fail, fail_msg,
                     known_exception_class=ImageComparisonFailure)
-                def do_test():
+                def do_test(fignum, actual_fname, expected_fname):
                     figure = plt.figure(fignum)
 
                     if self._remove_text:
@@ -255,7 +265,7 @@ def do_test():
                                 (self._freetype_version, ft2font.__freetype_version__))
                         raise
 
-                yield (do_test,)
+                yield do_test, fignum, actual_fname, expected_fname
 
 
 def image_comparison(baseline_images=None, extensions=None, tol=0,

diff --git a/lib/matplotlib/tests/test_axes.py b/lib/matplotlib/tests/test_axes.py
@@ -22,7 +22,8 @@
 import warnings
 
 import matplotlib
-from matplotlib.testing.decorators import image_comparison, cleanup, skipif
+from matplotlib.testing.decorators import image_comparison, cleanup
+from matplotlib.testing import skip
 import matplotlib.pyplot as plt
 import matplotlib.markers as mmarkers
 import matplotlib.patches as mpatches
@@ -86,11 +87,11 @@ def test_formatter_ticker():
     ax.autoscale_view()
 
 
-@skipif(LooseVersion(np.__version__) >= LooseVersion('1.11.0'),
-        reason="Fall out from a fixed numpy bug")
 @image_comparison(baseline_images=["formatter_large_small"])
 def test_formatter_large_small():
     # github issue #617, pull #619
+    if LooseVersion(np.__version__) >= LooseVersion('1.11.0'):
+        skip("Fall out from a fixed numpy bug")
     fig, ax = plt.subplots(1)
     x = [0.500000001, 0.500000002]
     y = [1e64, 1.1e64]