Merge pull request #151 from maxirmx/master

Appveyor reusable code

.appveyor.yml

@@ -18,36 +18,27 @@ environment:
 
     # The lastest Python 3.4.
     - PYTHON: "C:\\Python34-x64" 
-      PYTHON_VERSION: "3.4.x" # currently 3.4.3 
+      PYTHON_VERSION: "3.4.3" # currently 3.4.3 
       PYTHON_ARCH: "64" 
 
-    #- PYTHON: "C:\\Python34-x32" 
+    - PYTHON: "C:\\Python34-x32" 
-    #  PYTHON_VERSION: "3.4.x" # currently 3.4.3 
+      PYTHON_VERSION: "3.4.3" # currently 3.4.3 
-    #  PYTHON_ARCH: "32" 
+      PYTHON_ARCH: "32" 
 
 
 install:
+  # Checking submodule(s) out
+  - git submodule update --init --recursive
   # Install Python (from the official .msi of http://python.org) and pip when
   # not already installed.
-  - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
+  - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 -python -pip -stdint }
 
   # Prepend newly installed Python to the PATH of this build (this cannot be
   # done from inside the powershell script as it would require to restart
   # the parent CMD process).
-  - "SET PATH=%CD%;%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
   - "SET PYTHONPATH=%CD%;%PYTHONPATH%"
 
-  # Filesystem root
-  #- ps: "ls \"C:/\""
-  #- SET
-
-  # Installed SDKs
-  #- ps: "ls \"C:/Program Files/Microsoft SDKs/Windows\""
-
-  # Checking stdint.h
-  #- ps: "ls \"C:/projects/spacy/include/\""
-
-
   # Check that we have the expected version and architecture for Python
   - "python --version"
   - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
@@ -69,15 +60,13 @@ build_script:
   - "%CMD_IN_ENV% python setup.py build_ext --inplace"
   - ps: appveyor\download.ps1 
   - "tar -xzf corpora/en/wordnet.tar.gz"
-  #- "ls \"C:/projects/spacy/corpora/en/"
-  #- "ls \"C:/projects/spacy/"
   - "%CMD_IN_ENV% python bin/init_model.py en lang_data/ corpora/ spacy/en/data"
 
 
 test_script:
   # Run the project tests
   - "pip install pytest"
-  - "%CMD_IN_ENV% py.test tests/ -x"
+  - "%CMD_IN_ENV% py.test spacy/ -x"
 
 after_test:
   # If tests are successful, create binary packages for the project.

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "appveyor"]
+	path = appveyor
+	url = https://github.com/henningpeters/spaCy-appveyor-toolkit

README-MSVC.txt

@@ -9,5 +9,9 @@ Build steps:
 	- pip install cython fabric fabtools
 	- pip install -r requirements.txt
 	- python setup.py build_ext --inplace
+
+If you are using traditional Microsoft SDK (v7.0 for Python 2.x or v7.1 for Python 3.x) consider run_with_env.cmd from appveyor folder (submodule) as a guideline for environment setup. 
+It can be also used as shell conviguration script for your build, install and run commands, i.e.: cmd /E:ON /V:ON /C run_with_env.cmd <your command> 
+
  
  

appveyor

@@ -0,0 +1 @@
+Subproject commit 9f94a16f0e652e961af4ff911cffe819a8cb3e77

appveyor/download.ps1

@@ -1,51 +0,0 @@
-# Wordnet download Windows script
-
-$WORDNET_URL = "http://wordnetcode.princeton.edu/3.0/WordNet-3.0.tar.gz"
-$WORDNET_RELATIVE_PATH = "corpora\en"
-
-function Download ($filename, $url) {
-    $webclient = New-Object System.Net.WebClient
-
-    $basedir = $pwd.Path + "\"
-    $filepath = $basedir + $filename
-    if (Test-Path $filename) {
-        Write-Host "Reusing" $filepath
-        return $filepath
-    }
-                                                                               
-    # Download and retry up to 3 times in case of network transient errors.
-    Write-Host "Downloading" $filename "from" $url
-    $retry_attempts = 2
-    for ($i = 0; $i -lt $retry_attempts; $i++) {
-        try {
-            $webclient.DownloadFile($url, $filepath)
-            break
-        }
-        Catch [Exception]{
-            Start-Sleep 1
-        }
-    }
-    if (Test-Path $filepath) {
-        Write-Host "File saved at" $filepath
-    } else {
-        # Retry once to get the error message if any at the last try
-        $webclient.DownloadFile($url, $filepath)
-    }
-    return $filepath
-}
-
-function InstallWordNet () {
-   if((Test-Path $WORDNET_RELATIVE_PATH) -eq 0)
-   {
-       mkdir $WORDNET_RELATIVE_PATH;
-   }
-   $wordnet_fname = $WORDNET_RELATIVE_PATH + "\wordnet.tar.gz" 
-   Download  $wordnet_fname $WORDNET_URL
-}
-
-
-function main () {
-    InstallWordNet
-}
-
-main

appveyor/install.ps1

@@ -1,249 +0,0 @@
-# Based on the script to install Python and pip under Windows
-# Authors: Olivier Grisel, Jonathan Helmus, Kyle Kastner, and Alex Willmer
-# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
-#
-# Added: stdint.h download for Python 2.x
-
-$MINICONDA_URL = "http://repo.continuum.io/miniconda/"
-$BASE_URL = "https://www.python.org/ftp/python/"
-$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py"
-$GET_PIP_PATH = "C:\get-pip.py"
-$STDINT_H_URL = "http://msinttypes.googlecode.com/svn/trunk/stdint.h"
-$STDINT_H_RELATIVE_PATH = "include"
-
-$PYTHON_PRERELEASE_REGEX = @"
-(?x)
-(?<major>\d+)
-\.
-(?<minor>\d+)
-\.
-(?<micro>\d+)
-(?<prerelease>[a-z]{1,2}\d+)
-"@
-
-
-function Download ($filename, $url) {
-    $webclient = New-Object System.Net.WebClient
-
-    $basedir = $pwd.Path + "\"
-    $filepath = $basedir + $filename
-    if (Test-Path $filename) {
-        Write-Host "Reusing" $filepath
-        return $filepath
-    }
-
-    # Download and retry up to 3 times in case of network transient errors.
-    Write-Host "Downloading" $filename "from" $url
-    $retry_attempts = 2
-    for ($i = 0; $i -lt $retry_attempts; $i++) {
-        try {
-            $webclient.DownloadFile($url, $filepath)
-            break
-        }
-        Catch [Exception]{
-            Start-Sleep 1
-        }
-    }
-    if (Test-Path $filepath) {
-        Write-Host "File saved at" $filepath
-    } else {
-        # Retry once to get the error message if any at the last try
-        $webclient.DownloadFile($url, $filepath)
-    }
-    return $filepath
-}
-
-
-function ParsePythonVersion ($python_version) {
-    if ($python_version -match $PYTHON_PRERELEASE_REGEX) {
-        return ([int]$matches.major, [int]$matches.minor, [int]$matches.micro,
-                $matches.prerelease)
-    }
-    $version_obj = [version]$python_version
-    return ($version_obj.major, $version_obj.minor, $version_obj.build, "")
-}
-
-
-function DownloadPython ($python_version, $platform_suffix) {
-    $major, $minor, $micro, $prerelease = ParsePythonVersion $python_version
-
-    if (($major -le 2 -and $micro -eq 0) `
-        -or ($major -eq 3 -and $minor -le 2 -and $micro -eq 0) `
-        ) {
-        $dir = "$major.$minor"
-        $python_version = "$major.$minor$prerelease"
-    } else {
-        $dir = "$major.$minor.$micro"
-    }
-
-    if ($prerelease) {
-        if (($major -le 2) `
-            -or ($major -eq 3 -and $minor -eq 1) `
-            -or ($major -eq 3 -and $minor -eq 2) `
-            -or ($major -eq 3 -and $minor -eq 3) `
-            ) {
-            $dir = "$dir/prev"
-        }
-    }
-
-    if (($major -le 2) -or ($major -le 3 -and $minor -le 4)) {
-        $ext = "msi"
-        if ($platform_suffix) {
-            $platform_suffix = ".$platform_suffix"
-        }
-    } else {
-        $ext = "exe"
-        if ($platform_suffix) {
-            $platform_suffix = "-$platform_suffix"
-        }
-    }
-
-    $filename = "python-$python_version$platform_suffix.$ext"
-    $url = "$BASE_URL$dir/$filename"
-    $filepath = Download $filename $url
-    return $filepath
-}
-
-
-function InstallPython ($python_version, $architecture, $python_home) {
-    Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
-    if (Test-Path $python_home) {
-        Write-Host $python_home "already exists, skipping."
-        return $false
-    }
-    if ($architecture -eq "32") {
-        $platform_suffix = ""
-    } else {
-        $platform_suffix = "amd64"
-    }
-    $installer_path = DownloadPython $python_version $platform_suffix
-    $installer_ext = [System.IO.Path]::GetExtension($installer_path)
-    Write-Host "Installing $installer_path to $python_home"
-    $install_log = $python_home + ".log"
-    if ($installer_ext -eq '.msi') {
-        InstallPythonMSI $installer_path $python_home $install_log
-    } else {
-        InstallPythonEXE $installer_path $python_home $install_log
-    }
-    if (Test-Path $python_home) {
-        Write-Host "Python $python_version ($architecture) installation complete"
-    } else {
-        Write-Host "Failed to install Python in $python_home"
-        Get-Content -Path $install_log
-        Exit 1
-    }
-}
-
-
-function InstallPythonEXE ($exepath, $python_home, $install_log) {
-    $install_args = "/quiet InstallAllUsers=1 TargetDir=$python_home"
-    RunCommand $exepath $install_args
-}
-
-
-function InstallPythonMSI ($msipath, $python_home, $install_log) {
-    $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home"
-    $uninstall_args = "/qn /x $msipath"
-    RunCommand "msiexec.exe" $install_args
-    if (-not(Test-Path $python_home)) {
-        Write-Host "Python seems to be installed else-where, reinstalling."
-        RunCommand "msiexec.exe" $uninstall_args
-        RunCommand "msiexec.exe" $install_args
-    }
-}
-
-function RunCommand ($command, $command_args) {
-    Write-Host $command $command_args
-    Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru
-}
-
-
-function InstallPip ($python_home) {
-    $pip_path = $python_home + "\Scripts\pip.exe"
-    $python_path = $python_home + "\python.exe"
-    if (-not(Test-Path $pip_path)) {
-        Write-Host "Installing pip..."
-        $webclient = New-Object System.Net.WebClient
-        $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH)
-        Write-Host "Executing:" $python_path $GET_PIP_PATH
-        & $python_path $GET_PIP_PATH
-    } else {
-        Write-Host "pip already installed."
-    }
-}
-
-
-function DownloadMiniconda ($python_version, $platform_suffix) {
-    if ($python_version -eq "3.4") {
-        $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe"
-    } else {
-        $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe"
-    }
-    $url = $MINICONDA_URL + $filename
-    $filepath = Download $filename $url
-    return $filepath
-}
-
-
-function InstallMiniconda ($python_version, $architecture, $python_home) {
-    Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
-    if (Test-Path $python_home) {
-        Write-Host $python_home "already exists, skipping."
-        return $false
-    }
-    if ($architecture -eq "32") {
-        $platform_suffix = "x86"
-    } else {
-        $platform_suffix = "x86_64"
-    }
-    $filepath = DownloadMiniconda $python_version $platform_suffix
-    Write-Host "Installing" $filepath "to" $python_home
-    $install_log = $python_home + ".log"
-    $args = "/S /D=$python_home"
-    Write-Host $filepath $args
-    Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru
-    if (Test-Path $python_home) {
-        Write-Host "Python $python_version ($architecture) installation complete"
-    } else {
-        Write-Host "Failed to install Python in $python_home"
-        Get-Content -Path $install_log
-        Exit 1
-    }
-}
-
-
-function InstallMinicondaPip ($python_home) {
-    $pip_path = $python_home + "\Scripts\pip.exe"
-    $conda_path = $python_home + "\Scripts\conda.exe"
-    if (-not(Test-Path $pip_path)) {
-        Write-Host "Installing pip..."
-        $args = "install --yes pip"
-        Write-Host $conda_path $args
-        Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru
-    } else {
-        Write-Host "pip already installed."
-    }
-}
-
-function InstallStdintH ($python_version) {
-    $major, $minor, $micro, $prerelease = ParsePythonVersion $python_version
-    if ($major -le 2) {
-        if((Test-Path $STDINT_H_RELATIVE_PATH) -eq 0)
-        {
-            mkdir $STDINT_H_RELATIVE_PATH;
-        }
-        $stdint_h_fname = $STDINT_H_RELATIVE_PATH + "\stdint.h" 
-        Download  $stdint_h_fname $STDINT_H_URL
-    } else {
-        Write-Host $python_version " uses C99 compliant Microsoft compiler. stdint.h download is not required."
-    }
-}
-
-
-function main () {
-    InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON
-    InstallPip $env:PYTHON
-    InstallStdintH $env:PYTHON_VERSION
-}
-
-main

appveyor/run_with_env.cmd

@@ -1,94 +0,0 @@
-:: To build extensions for 64 bit Python 3, we need to configure environment
-:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
-:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1)
-::
-:: To build extensions for 64 bit Python 2, we need to configure environment
-:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of:
-:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0)
-::
-:: To provide C99 compatibility C:\projects\include is added to INCLUDE environment variable
-:: in a believe that stdin.h has been uploaded there earlier during the installation
-::
-:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific
-:: environment configurations.
-::
-:: Note: this script needs to be run with the /E:ON and /V:ON flags for the
-:: cmd interpreter, at least for (SDK v7.0)
-::
-:: More details at:
-:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
-:: http://stackoverflow.com/a/13751649/163740
-::
-:: Author: Olivier Grisel
-:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
-::
-:: Notes about batch files for Python people:
-::
-:: Quotes in values are literally part of the values:
-::      SET FOO="bar"
-:: FOO is now five characters long: " b a r "
-:: If you don't want quotes, don't include them on the right-hand side.
-::
-:: The CALL lines at the end of this file look redundant, but if you move them
-:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y
-:: case, I don't know why.
-@ECHO OFF
-
-SET COMMAND_TO_RUN=%*
-SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows
-SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf
-
-:: Extract the major and minor versions, and allow for the minor version to be
-:: more than 9.  This requires the version number to have two dots in it.
-SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1%
-IF "%PYTHON_VERSION:~3,1%" == "." (
-    SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1%
-) ELSE (
-    SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2%
-)
-
-:: Based on the Python version, determine what SDK version to use, and whether
-:: to set the SDK for 64-bit.
-:: For Python 2.x INCLUDE environment variable is supplemented by local include directory
-:: which is supposed to contain C99 compliant stdint.h
-IF %MAJOR_PYTHON_VERSION% == 2 (
-    SET WINDOWS_SDK_VERSION="v7.0"
-    SET SET_SDK_64=Y
-    SET INCLUDE=%CD%\include;%INCLUDE%
-) ELSE (
-    IF %MAJOR_PYTHON_VERSION% == 3 (
-        SET WINDOWS_SDK_VERSION="v7.1"
-        IF %MINOR_PYTHON_VERSION% LEQ 4 (
-            SET SET_SDK_64=Y
-        ) ELSE (
-            SET SET_SDK_64=N
-            IF EXIST "%WIN_WDK%" (
-                :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/
-                REN "%WIN_WDK%" 0wdf
-            )
-        )
-    ) ELSE (
-        ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
-        EXIT 1
-    )
-)
-
-IF %PYTHON_ARCH% == 64 (
-    IF %SET_SDK_64% == Y (
-        ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
-        SET DISTUTILS_USE_SDK=1
-        SET MSSdk=1
-        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
-        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
-        ECHO Executing: %COMMAND_TO_RUN%
-        call %COMMAND_TO_RUN% || EXIT 1
-    ) ELSE (
-        ECHO Using default MSVC build environment for 64 bit architecture
-        ECHO Executing: %COMMAND_TO_RUN%
-        call %COMMAND_TO_RUN% || EXIT 1
-    )
-) ELSE (
-    ECHO Using default MSVC build environment for 32 bit architecture
-    ECHO Executing: %COMMAND_TO_RUN%
-    call %COMMAND_TO_RUN% || EXIT 1
-)