From 2633a26f60e125ab7db9591ed6f006fa79532080 Mon Sep 17 00:00:00 2001 From: Zethson Date: Wed, 20 Jan 2021 13:26:08 +0100 Subject: [PATCH 01/12] add twine check to azure CI + 3.8 and 3.9 matrix Signed-off-by: Zethson --- .azure-pipelines.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 5bbfe90779..0bf9e59a8f 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -11,6 +11,10 @@ jobs: vmImage: 'ubuntu-18.04' strategy: matrix: + Python39: + python.version: '3.9' + Python38: + python.version: '3.8' Python37: python.version: '3.7' Python36: @@ -79,3 +83,9 @@ jobs: python setup.py check --restructuredtext --strict rst2html.py --halt=2 README.rst >/dev/null displayName: 'rst2html' + + - script: | + pip install setuptools twine + python setup.py sdist bdist_wheel + twine check dist/* + displayName: 'Twine check' From 84e3007bf8a8b37f472d8e7d739c4ffcc855581b Mon Sep 17 00:00:00 2001 From: Zethson Date: Wed, 20 Jan 2021 13:40:44 +0100 Subject: [PATCH 02/12] remove 3.9 from azure matrix Signed-off-by: Zethson --- .azure-pipelines.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 0bf9e59a8f..92c9cf8ab5 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -11,8 +11,6 @@ jobs: vmImage: 'ubuntu-18.04' strategy: matrix: - Python39: - python.version: '3.9' Python38: python.version: '3.8' Python37: From fb81e84aac557c952d2d18654f244c80f93ef22e Mon Sep 17 00:00:00 2001 From: Zethson Date: Wed, 20 Jan 2021 13:45:41 +0100 Subject: [PATCH 03/12] removed 3.6 from azure CI Signed-off-by: Zethson --- .azure-pipelines.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 92c9cf8ab5..7dc4288627 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -15,8 +15,6 @@ jobs: python.version: '3.8' Python37: python.version: '3.7' - Python36: - python.version: '3.6' anndata_dev: python.version: '3.7' ANNDATA_DEV: yes From 94eb303b0d15f6ab88181438264358b3ac8fd4db Mon Sep 17 00:00:00 2001 From: Zethson Date: Wed, 20 Jan 2021 14:18:12 +0100 Subject: [PATCH 04/12] add 3.6 to CI, remove 3.7 from CI Signed-off-by: Zethson --- .azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 7dc4288627..d2def7e3d6 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -14,9 +14,9 @@ jobs: Python38: python.version: '3.8' Python37: - python.version: '3.7' + python.version: '3.6' anndata_dev: - python.version: '3.7' + python.version: '3.8' ANNDATA_DEV: yes RUN_COVERAGE: yes steps: From 8d56baecf669107ddd77c7a0e4549e990840a0c4 Mon Sep 17 00:00:00 2001 From: Zethson Date: Thu, 21 Jan 2021 10:06:57 +0100 Subject: [PATCH 05/12] Moved Twine check into separate job + removed setup.py rst check Signed-off-by: Zethson --- .azure-pipelines.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index d2def7e3d6..a0cc6da220 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -76,12 +76,19 @@ jobs: testRunTitle: 'Publish test results for Python $(python.version)' - script: | - python setup.py check --restructuredtext --strict rst2html.py --halt=2 README.rst >/dev/null displayName: 'rst2html' - - script: | - pip install setuptools twine +- job: Twine Check + pool: + vmImage: 'ubuntu-18.04' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.8' + + - script: | + pip install setuptools twine wheel python setup.py sdist bdist_wheel twine check dist/* displayName: 'Twine check' From 03a9384c91c90b26eb7ea35ef27169c40d8c06b7 Mon Sep 17 00:00:00 2001 From: Zethson Date: Thu, 21 Jan 2021 10:29:08 +0100 Subject: [PATCH 06/12] fix twine check intendation Signed-off-by: Zethson --- .azure-pipelines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index a0cc6da220..824f4e8d7b 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -79,7 +79,7 @@ jobs: rst2html.py --halt=2 README.rst >/dev/null displayName: 'rst2html' -- job: Twine Check +- job: Twine pool: vmImage: 'ubuntu-18.04' steps: @@ -88,7 +88,7 @@ jobs: versionSpec: '3.8' - script: | - pip install setuptools twine wheel - python setup.py sdist bdist_wheel - twine check dist/* - displayName: 'Twine check' + pip install setuptools twine wheel + python setup.py sdist bdist_wheel + twine check dist/* + displayName: 'Twine check' From ae06c7b37b58b397d1a1a8e9540d8bc32da820aa Mon Sep 17 00:00:00 2001 From: Zethson Date: Thu, 21 Jan 2021 10:41:12 +0100 Subject: [PATCH 07/12] remove extra job and use condition for 3.8 twine check Signed-off-by: Zethson --- .azure-pipelines.yml | 145 ++++++++++++++++++++----------------------- 1 file changed, 69 insertions(+), 76 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 824f4e8d7b..cbcfe39b42 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -1,94 +1,87 @@ trigger: -- master + - master -variables: - ANNDATA_DEV: no - RUN_COVERAGE: no + variables: + ANNDATA_DEV: no + RUN_COVERAGE: no -jobs: -- job: PyTest - pool: - vmImage: 'ubuntu-18.04' - strategy: - matrix: - Python38: - python.version: '3.8' - Python37: - python.version: '3.6' - anndata_dev: - python.version: '3.8' - ANNDATA_DEV: yes - RUN_COVERAGE: yes - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - displayName: 'Use Python $(python.version)' - - - script: | - export MPLBACKEND="agg" - echo $MPLBACKEND - displayName: 'Set env' - - - script: | - python -m pip install --upgrade pip - pip install pytest-cov wheel - pip install -e .[dev,doc,test,louvain,leiden,magic,scvi,harmony,scrublet,scanorama] - displayName: 'Install dependencies' + jobs: + - job: PyTest + pool: + vmImage: 'ubuntu-18.04' + strategy: + matrix: + Python38: + python.version: '3.8' + Python37: + python.version: '3.6' + anndata_dev: + python.version: '3.8' + ANNDATA_DEV: yes + RUN_COVERAGE: yes + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' - - script: | - pip install -v git+https://github.com/theislab/anndata - displayName: 'Install development anndata' - condition: eq(variables['ANNDATA_DEV'], 'yes') + - script: | + export MPLBACKEND="agg" + echo $MPLBACKEND + displayName: 'Set env' - - script: | - pip list - displayName: 'Display installed versions' + - script: | + python -m pip install --upgrade pip + pip install pytest-cov wheel + pip install -e .[dev,doc,test,louvain,leiden,magic,scvi,harmony,scrublet,scanorama] + displayName: 'Install dependencies' - - script: | - pip install black - black . --check --diff - python -m scanpy.tests.blackdiff 10 - displayName: 'Black' + - script: | + pip install -v git+https://github.com/theislab/anndata + displayName: 'Install development anndata' + condition: eq(variables['ANNDATA_DEV'], 'yes') - - script: | - pytest --color=yes --ignore=scanpy/tests/_images --nunit-xml="nunit/test-results.xml" - displayName: 'PyTest' - condition: eq(variables['RUN_COVERAGE'], 'no') + - script: | + pip list + displayName: 'Display installed versions' - - script: | - pytest --color=yes --ignore=scanpy/tests/_images --nunit-xml="nunit/test-results.xml" --cov=scanpy --cov-report=xml - displayName: 'PyTest (coverage)' - condition: eq(variables['RUN_COVERAGE'], 'yes') + - script: | + pip install black + black . --check --diff + python -m scanpy.tests.blackdiff 10 + displayName: 'Black' - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' - reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' - condition: eq(variables['RUN_COVERAGE'], 'yes') + - script: | + pytest --color=yes --ignore=scanpy/tests/_images --nunit-xml="nunit/test-results.xml" + displayName: 'PyTest' + condition: eq(variables['RUN_COVERAGE'], 'no') - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: 'nunit/test-results.xml' - testResultsFormat: NUnit - testRunTitle: 'Publish test results for Python $(python.version)' + - script: | + pytest --color=yes --ignore=scanpy/tests/_images --nunit-xml="nunit/test-results.xml" --cov=scanpy --cov-report=xml + displayName: 'PyTest (coverage)' + condition: eq(variables['RUN_COVERAGE'], 'yes') - - script: | - rst2html.py --halt=2 README.rst >/dev/null - displayName: 'rst2html' + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' + reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' + condition: eq(variables['RUN_COVERAGE'], 'yes') -- job: Twine - pool: - vmImage: 'ubuntu-18.04' - steps: - - task: UsePythonVersion@0 + - task: PublishTestResults@2 + condition: succeededOrFailed() inputs: - versionSpec: '3.8' + testResultsFiles: 'nunit/test-results.xml' + testResultsFormat: NUnit + testRunTitle: 'Publish test results for Python $(python.version)' + + - script: | + rst2html.py --halt=2 README.rst >/dev/null + displayName: 'rst2html' - script: | - pip install setuptools twine wheel + pip install setuptools twine python setup.py sdist bdist_wheel twine check dist/* + condition: in(variables['python.version'], '3.8') displayName: 'Twine check' From c00ef817bda02f18526ef32a52708d33b7c53251 Mon Sep 17 00:00:00 2001 From: Zethson Date: Thu, 21 Jan 2021 10:46:02 +0100 Subject: [PATCH 08/12] fix CI formatting Signed-off-by: Zethson --- .azure-pipelines.yml | 147 ++++++++++++++++++++++--------------------- 1 file changed, 74 insertions(+), 73 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index cbcfe39b42..831450d315 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -1,87 +1,88 @@ trigger: - - master +- master - variables: - ANNDATA_DEV: no - RUN_COVERAGE: no +variables: + ANNDATA_DEV: no + RUN_COVERAGE: no - jobs: - - job: PyTest - pool: - vmImage: 'ubuntu-18.04' - strategy: - matrix: - Python38: - python.version: '3.8' - Python37: - python.version: '3.6' - anndata_dev: - python.version: '3.8' - ANNDATA_DEV: yes - RUN_COVERAGE: yes - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - displayName: 'Use Python $(python.version)' +jobs: +- job: PyTest + pool: + vmImage: 'ubuntu-18.04' + strategy: + matrix: + Python38: + python.version: '3.8' + Python36: + python.version: '3.6' + anndata_dev: + python.version: '3.8' + ANNDATA_DEV: yes + RUN_COVERAGE: yes + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' - - script: | - export MPLBACKEND="agg" - echo $MPLBACKEND - displayName: 'Set env' + - script: | + export MPLBACKEND="agg" + echo $MPLBACKEND + displayName: 'Set env' - - script: | - python -m pip install --upgrade pip - pip install pytest-cov wheel - pip install -e .[dev,doc,test,louvain,leiden,magic,scvi,harmony,scrublet,scanorama] - displayName: 'Install dependencies' + - script: | + python -m pip install --upgrade pip + pip install pytest-cov wheel + pip install -e .[dev,doc,test,louvain,leiden,magic,scvi,harmony,scrublet,scanorama] + displayName: 'Install dependencies' - - script: | - pip install -v git+https://github.com/theislab/anndata - displayName: 'Install development anndata' - condition: eq(variables['ANNDATA_DEV'], 'yes') + - script: | + pip install -v git+https://github.com/theislab/anndata + displayName: 'Install development anndata' + condition: eq(variables['ANNDATA_DEV'], 'yes') - - script: | - pip list - displayName: 'Display installed versions' + - script: | + pip list + displayName: 'Display installed versions' - - script: | - pip install black - black . --check --diff - python -m scanpy.tests.blackdiff 10 - displayName: 'Black' + - script: | + pip install black + black . --check --diff + python -m scanpy.tests.blackdiff 10 + displayName: 'Black' - - script: | - pytest --color=yes --ignore=scanpy/tests/_images --nunit-xml="nunit/test-results.xml" - displayName: 'PyTest' - condition: eq(variables['RUN_COVERAGE'], 'no') + - script: | + pytest --color=yes --ignore=scanpy/tests/_images --nunit-xml="nunit/test-results.xml" + displayName: 'PyTest' + condition: eq(variables['RUN_COVERAGE'], 'no') - - script: | - pytest --color=yes --ignore=scanpy/tests/_images --nunit-xml="nunit/test-results.xml" --cov=scanpy --cov-report=xml - displayName: 'PyTest (coverage)' - condition: eq(variables['RUN_COVERAGE'], 'yes') + - script: | + pytest --color=yes --ignore=scanpy/tests/_images --nunit-xml="nunit/test-results.xml" --cov=scanpy --cov-report=xml + displayName: 'PyTest (coverage)' + condition: eq(variables['RUN_COVERAGE'], 'yes') - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' - reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' - condition: eq(variables['RUN_COVERAGE'], 'yes') + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' + reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' + condition: eq(variables['RUN_COVERAGE'], 'yes') - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: 'nunit/test-results.xml' - testResultsFormat: NUnit - testRunTitle: 'Publish test results for Python $(python.version)' + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: 'nunit/test-results.xml' + testResultsFormat: NUnit + testRunTitle: 'Publish test results for Python $(python.version)' - - script: | - rst2html.py --halt=2 README.rst >/dev/null - displayName: 'rst2html' + - script: | + rst2html.py --halt=2 README.rst >/dev/null + displayName: 'rst2html' - - script: | - pip install setuptools twine - python setup.py sdist bdist_wheel - twine check dist/* - condition: in(variables['python.version'], '3.8') - displayName: 'Twine check' + - script: | + python -m pip install --upgrade pip + pip install setuptools twine wheel + python setup.py sdist bdist_wheel + twine check dist/* + condition: in(variables['python.version'], '3.8') + displayName: 'Twine check' From a888a30071717cfe8c37af280ca8254c9210873f Mon Sep 17 00:00:00 2001 From: Zethson Date: Thu, 21 Jan 2021 10:47:10 +0100 Subject: [PATCH 09/12] removed wheel installation for twine check Signed-off-by: Zethson --- .azure-pipelines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 831450d315..b55186ad6a 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -80,8 +80,7 @@ jobs: displayName: 'rst2html' - script: | - python -m pip install --upgrade pip - pip install setuptools twine wheel + pip install setuptools twine python setup.py sdist bdist_wheel twine check dist/* condition: in(variables['python.version'], '3.8') From 676dcb6e109593ed8218e7006594080922f93fdb Mon Sep 17 00:00:00 2001 From: Isaac Virshup Date: Mon, 25 Jan 2021 11:50:54 +1100 Subject: [PATCH 10/12] Build and twine check in separate job --- .azure-pipelines.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index b55186ad6a..97ece237aa 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -75,13 +75,25 @@ jobs: testResultsFormat: NUnit testRunTitle: 'Publish test results for Python $(python.version)' - - script: | - rst2html.py --halt=2 README.rst >/dev/null - displayName: 'rst2html' +- job: CheckBuild + pool: + vmImage: 'ubuntu-18.04' + steps: + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.8' + displayName: 'Use Python 3.8' - script: | + python -m pip install --upgrade pip pip install setuptools twine + displayName: 'Install dependencies' + + - script: pip list + displayName: 'Display installed versions' + + - script: | python setup.py sdist bdist_wheel twine check dist/* - condition: in(variables['python.version'], '3.8') - displayName: 'Twine check' + displayName: 'Build & Twine check' From 22f2ea1b958e627296c3f40e5bde2a655f242a2c Mon Sep 17 00:00:00 2001 From: Isaac Virshup Date: Mon, 25 Jan 2021 12:29:11 +1100 Subject: [PATCH 11/12] Add pytoml build dependency --- .azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 97ece237aa..8bbf7d16a6 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -87,7 +87,7 @@ jobs: - script: | python -m pip install --upgrade pip - pip install setuptools twine + pip install setuptools twine pytoml displayName: 'Install dependencies' - script: pip list From a1da783db9b991caa72ebaa3247be1242a360b89 Mon Sep 17 00:00:00 2001 From: Isaac Virshup Date: Mon, 25 Jan 2021 12:34:12 +1100 Subject: [PATCH 12/12] Add other build dependencies (someday maybe we can parse these out?) --- .azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 8bbf7d16a6..80996bf430 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -87,8 +87,8 @@ jobs: - script: | python -m pip install --upgrade pip - pip install setuptools twine pytoml - displayName: 'Install dependencies' + pip install setuptools setuptools_scm pytoml wheel twine + displayName: 'Install build dependencies' - script: pip list displayName: 'Display installed versions'