streamline ci/build systems

.appveyor.yml

@@ -43,30 +43,20 @@ install:
   - "python --version"
   - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
 
-  # Upgrade to the latest version of pip to avoid it displaying warnings
-  # about it being out of date.
-  - "pip install --disable-pip-version-check --user --upgrade pip"
-
   # Install the build dependencies of the project. If some dependencies contain
   # compiled extensions and are not provided as pre-built wheel packages,
   # pip will build them from source using the MSVC compiler matching the
   # target Python version and architecture
-  - "pip install --upgrade setuptools"
+  - "%CMD_IN_ENV% python build.py prepare"
-  - "%CMD_IN_ENV% pip install cython fabric fabtools"
-  - "%CMD_IN_ENV% pip install -r requirements.txt"
 
 build_script:
   # Build the compiled extension
-  - "%CMD_IN_ENV% python setup.py build_ext --inplace"
+  - "%CMD_IN_ENV% python build.py pip"
-  - ps: appveyor\download.ps1 
-  - "tar -xzf corpora/en/wordnet.tar.gz"
-  - "%CMD_IN_ENV% python bin/init_model.py en lang_data/ corpora/ spacy/en/data"
 
 
 test_script:
   # Run the project tests
-  - "pip install pytest"
+  - "%CMD_IN_ENV% python build.py test"
-  - "%CMD_IN_ENV% py.test spacy/ -x"
 
 after_test:
   # If tests are successful, create binary packages for the project.

.travis.yml

@@ -17,28 +17,16 @@ env:
   - PIP_DATE=2015-10-01 PIP=0 INSTALL=0 DEVELOP=1
 
 install:
-  - pip install -U pip
+  - python build.py prepare $PIP_DATE
-  - python pip-date.py $PIP_DATE pip setuptools wheel six
-  - pip install -r requirements.txt
-
-  - if [[ "$PIP" == "1" ]]; then
-      python setup.py sdist;
-      pip install dist/*;
-    fi
-
-  - if [[ "$INSTALL" == "1" ]]; then
-      python setup.py install;
-    fi
-
-  - if [[ "$DEVELOP" == "1" ]]; then
-      python setup.py develop;
-      pip install -e .;
-    fi
-
-  - pip install pytest
-  - pip list
 
 script:
-  - mkdir tmp; cd tmp
+  - if [[ "$PIP" == "1" ]]; then
-  - python -m spacy.en.download
+      python build.py pip;
-  - python -m pytest ../spacy/ -x --models --vectors --slow
+    fi
+  - if [[ "$INSTALL" == "1" ]]; then
+      python build.py setup-install;
+    fi
+  - if [[ "$DEVELOP" == "1" ]]; then
+      python build.py setup-develop;
+    fi
+  - python build.py test

build.py

@@ -19,41 +19,58 @@ if len(sys.argv) < 2:
 
 
 install_mode = sys.argv[1]
-pip_date = len(sys.argv) > 2 and sys.argv[2]
 
 
-x('pip install -U pip')
+if install_mode == 'prepare':
-if pip_date:
+    x('python pip-clear.py')
-    x('python pip-date.py %s pip setuptools wheel six' % pip_date)
+    x('pip install --disable-pip-version-check -U pip setuptools')
-x('pip install -r requirements.txt')
 
-if install_mode == 'pip':
+    pip_date = len(sys.argv) > 2 and sys.argv[2]
-    for filename in os.listdir('dist'):
+    if pip_date:
-        os.unlink(os.path.join('dist', filename))
+        x('python pip-date.py %s pip setuptools wheel six' % pip_date)
+
+    x('pip install -r requirements.txt')
+    x('pip list')
+
+
+elif install_mode == 'pip':
+    if os.path.exists('dist'):
+        shutil.rmtree('dist')
     x('python setup.py sdist')
+    x('python pip-clear.py')
+
     filenames = os.listdir('dist')
     assert len(filenames) == 1
+    x('pip list')
     x('pip install dist/%s' % filenames[0])
 
+
 elif install_mode == 'setup-install':
     x('python setup.py install')
 
+
 elif install_mode == 'setup-develop':
+    x('python setup.py develop')
+    x('python pip-clear.py')
+
+    x('pip list')
     x('pip install -e .')
 
-x('pip install pytest')
-x('pip list')
 
-if os.path.exists('tmp'):
+elif install_mode == 'test':
-    shutil.rmtree('tmp')
+    x('pip install pytest')
-os.mkdir('tmp')
+    x('pip list')
 
-try:
+    if os.path.exists('tmp'):
-    old = os.getcwd()
+        shutil.rmtree('tmp')
-    os.chdir('tmp')
+    os.mkdir('tmp')
 
-    x('python -m spacy.en.download')
+    try:
-    x('python -m pytest ../spacy/ --models --vectors --slow')
+        old = os.getcwd()
+        os.chdir('tmp')
 
-finally:
+        x('python -m spacy.en.download')
-    os.chdir(old)
+        x('python -m pytest ../spacy/ --models --vectors --slow')
+
+    finally:
+        os.chdir(old)

pip-clear.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+from pip.commands.uninstall import UninstallCommand
+from pip import get_installed_distributions
+
+
+packages = []
+for package in get_installed_distributions():
+    if package.location.endswith('dist-packages'):
+        continue
+    elif package.project_name in ('pip', 'setuptools'):
+        continue
+    packages.append(package.project_name)
+
+
+if packages:
+    pip = UninstallCommand()
+    options, args = pip.parse_args(packages)
+    options.yes = True
+
+    try:
+        pip.run(options, args)
+    except OSError as e:
+        if e.errno != 13:
+            raise e
+        print("You lack permissions to uninstall this package. Perhaps run with sudo? Exiting.")
+        exit(13)

venv.ps1

@@ -10,5 +10,7 @@ if(!(Test-Path -Path ".build"))
 }
 .build\Scripts\activate.ps1
 
-python build.py $install_mode $pip_date
+python build.py prepare $pip_date
+python build.py $install_mode
+python build.py test
 exit $LASTEXITCODE

venv.sh

@@ -6,4 +6,6 @@ if [ ! -d ".build" ]; then
 fi
 
 . .build/bin/activate
-python build.py $2 $3
+python build.py prepare $3
+python build.py $2
+python build.py test