From cc752cf790823d3aeae7d3bc5ace4d202c00da41 Mon Sep 17 00:00:00 2001 From: Josh Peak Date: Tue, 6 Aug 2019 16:41:42 +1000 Subject: [PATCH] Collect and aggregate code coverage across all test environments (#460) * Collect and aggregate code coverage across all test environments Add Coveralls badge to readme Try CodeCov integration Setup coverage badge for CodeCov Fix CodeCov Badge * Set code coverage regression threshold Attempt to cache bust some boto3 issues on TravisCI * Disable legacy secure strings * Skip boto3 tests on TRAVIS_PULL_REQUEST due to credentials being unavailable Address flake8 issues Debug TRAVIS_PULL_REQUEST ENV Var Trying to get boto3 tests to skip on forked PRs boto3 is not an allowed failure remove TRAVIS ENV debugging * remove pytest skipif that wasn't working * Ignore coverage files --- .codecov.yml | 7 +++++++ .gitignore | 1 + .travis.yml | 11 ++++++----- README.rst | 6 +++++- tests/integration/test_boto3.py | 13 ++++++++++++- tox.ini | 27 +++++++++++++++++++++++++-- 6 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000..3a85288 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,7 @@ +coverage: + status: + project: + default: + target: 75 + # Allow 0% coverage regression + threshold: 0 diff --git a/.gitignore b/.gitignore index 30f7358..45702da 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ build/ dist/ *.egg/ .coverage +coverage.xml htmlcov/ *.egg-info/ pytestdebug.log diff --git a/.travis.yml b/.travis.yml index 50848d6..ff5482f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ language: python before_install: openssl version env: - global: - - secure: AifoKzwhjV94cmcQZrdQmqRu/9rkZZvWpwBv1daeAQpLOKFPGsOm3D+x2cSw9+iCfkgDZDfqQVv1kCaFVxTll8v8jTq5SJdqEY0NmGWbj/UkNtShh609oRDsuzLxAEwtVKYjf/h8K2BRea+bl1tGkwZ2vtmYS6dxNlAijjWOfds= - - secure: LBSEg/gMj4u4Hrpo3zs6Y/1mTpd2RtcN49mZIFgTdbJ9IhpiNPqcEt647Lz94F9Eses2x2WbNuKqZKZZReY7QLbEzU1m0nN5jlaKrjcG5NR5clNABfFFyhgc0jBikyS4abAG8jc2efeaTrFuQwdoF4sE8YiVrkiVj2X5Xoi6sBk= + # global: + # - secure: AifoKzwhjV94cmcQZrdQmqRu/9rkZZvWpwBv1daeAQpLOKFPGsOm3D+x2cSw9+iCfkgDZDfqQVv1kCaFVxTll8v8jTq5SJdqEY0NmGWbj/UkNtShh609oRDsuzLxAEwtVKYjf/h8K2BRea+bl1tGkwZ2vtmYS6dxNlAijjWOfds= + # - secure: LBSEg/gMj4u4Hrpo3zs6Y/1mTpd2RtcN49mZIFgTdbJ9IhpiNPqcEt647Lz94F9Eses2x2WbNuKqZKZZReY7QLbEzU1m0nN5jlaKrjcG5NR5clNABfFFyhgc0jBikyS4abAG8jc2efeaTrFuQwdoF4sE8YiVrkiVj2X5Xoi6sBk= matrix: - TOX_SUFFIX="flakes" - TOX_SUFFIX="requests" @@ -33,7 +33,6 @@ matrix: python: 3.7 dist: xenial allow_failures: - - env: TOX_SUFFIX="boto3" - env: TOX_SUFFIX="aiohttp" python: "pypy3.5-5.9.0" exclude: @@ -55,7 +54,9 @@ python: - pypy - "pypy3.5-5.9.0" install: -- pip install tox-travis +- pip install tox-travis codecov - if [[ $TOX_SUFFIX != 'flakes' ]]; then python setup.py install ; fi script: - tox -e "${TOX_SUFFIX}" +after_success: + - codecov diff --git a/README.rst b/README.rst index 8fcf330..b7ec219 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -|PyPI| |Python versions| |Build Status| |Gitter| +|PyPI| |Python versions| |Build Status| |CodeCov| |Gitter| VCR.py ====== @@ -56,3 +56,7 @@ more details .. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg :alt: Join the chat at https://gitter.im/kevin1024/vcrpy :target: https://gitter.im/kevin1024/vcrpy?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge +.. |CodeCov| image:: https://codecov.io/gh/kevin1024/vcrpy/branch/master/graph/badge.svg + :target: https://codecov.io/gh/kevin1024/vcrpy + :alt: Code Coverage Status + diff --git a/tests/integration/test_boto3.py b/tests/integration/test_boto3.py index 22e014a..078bb0e 100644 --- a/tests/integration/test_boto3.py +++ b/tests/integration/test_boto3.py @@ -26,7 +26,6 @@ boto3_skip_awsrequest = pytest.mark.skipif( reason='botocore version {ver} still uses vendored requests.'.format( ver=botocore.__version__)) - IAM_USER_NAME = "vcrpy" @@ -69,6 +68,12 @@ def test_boto_vendored_stubs(tmpdir): VerifiedHTTPSConnection('hostname.does.not.matter') +@pytest.mark.skipif( + os.environ.get("TRAVIS_PULL_REQUEST") != "false", + reason="Encrypted Environment Variables from Travis Repository Settings" + " are disabled on PRs from forks. " + "https://docs.travis-ci.com/user/pull-requests/#pull-requests-and-security-restrictions" +) def test_boto_medium_difficulty(tmpdir, get_user): with vcr.use_cassette(str(tmpdir.join('boto3-medium.yml'))): @@ -81,6 +86,12 @@ def test_boto_medium_difficulty(tmpdir, get_user): assert cass.all_played +@pytest.mark.skipif( + os.environ.get("TRAVIS_PULL_REQUEST") != "false", + reason="Encrypted Environment Variables from Travis Repository Settings" + " are disabled on PRs from forks. " + "https://docs.travis-ci.com/user/pull-requests/#pull-requests-and-security-restrictions" +) def test_boto_hardcore_mode(tmpdir, iam_client, get_user): with vcr.use_cassette(str(tmpdir.join('boto3-hardcore.yml'))): ses = boto3.Session( diff --git a/tox.ini b/tox.ini index a55904b..40be88f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,21 @@ [tox] -envlist = {py27,py35,py36,py37,pypy}-{flakes,requests,httplib2,urllib3,tornado4,boto3},{py35,py36,py37}-{aiohttp} +skip_missing_interpreters=true +envlist = cov-clean,{py27,py35,py36,py37,pypy}-{flakes,requests,httplib2,urllib3,tornado4,boto3},{py35,py36,py37}-{aiohttp},cov-report + + +# Coverage environment tasks: cov-clean and cov-report +# https://pytest-cov.readthedocs.io/en/latest/tox.html +[testenv:cov-clean] +deps = coverage +skip_install=true +commands = coverage erase + +[testenv:cov-report] +deps = coverage +skip_install=true +commands = + coverage html + coverage report --fail-under=75 [testenv:flakes] skipsdist = True @@ -10,8 +26,11 @@ commands = deps = flake8 [testenv] +# Need to use develop install so that paths +# for aggregate code coverage combine +usedevelop=true commands = - ./runtests.sh --cov={envsitepackagesdir}/vcr --cov-branch {posargs} + ./runtests.sh --cov=./vcr --cov-branch --cov-report=xml --cov-append {posargs} deps = Flask mock @@ -31,9 +50,13 @@ deps = aiohttp: aiohttp aiohttp: pytest-asyncio aiohttp: pytest-aiohttp +depends = + {py27,py35,py36,py37,pypy}-{flakes,requests,httplib2,urllib3,tornado4,boto3},{py35,py36,py37}-{aiohttp}: cov-clean + cov-report: {py27,py35,py36,py37,pypy}-{flakes,requests,httplib2,urllib3,tornado4,boto3},{py35,py36,py37}-{aiohttp} passenv = AWS_ACCESS_KEY_ID AWS_DEFAULT_REGION AWS_SECRET_ACCESS_KEY + [flake8] max_line_length = 110