Added script to implement appveyor functionality in Python

Only the init step for the moment.

.appveyor.yml

@@ -6,8 +6,6 @@ environment:
     global:
         # MSVC Express 2008's setenv.cmd failes if /E:ON and /V:ON are not
         # enabled in the batch script interpreter
-        #
-        #      WITH_COMPILER: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_compiler.cmd"
         CMD_IN_ENV: cmd /E:ON /V:ON /C .\appveyor\run_with_env.cmd
 
     matrix:
@@ -29,13 +27,14 @@ environment:
 
     PSYCOPG2_TESTDB: psycopg2_test
     PSYCOPG2_TESTDB_USER: postgres
-    PSYCOPG2_TESTDB_PASSWORD: Password12!
     PSYCOPG2_TESTDB_HOST: localhost
-    PSYCOPG2_TESTDB_PORT: 5432
 
     PGUSER: postgres
     PGPASSWORD: Password12!
 
+    # The python used in the build process, not the one packages are built for
+    PYEXE: C:\Python36\python.exe
+
 matrix:
     fast_finish: false
 
@@ -51,8 +50,12 @@ cache:
 
 # Script called before repo cloning
 init:
-    # Uncomment next line to get RDP access during the build.
+    # TODO: move functionalities out of init to drop this fetch problem
-    #- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+    - curl -fsSL -o "C:\\appveyor.py" https://raw.githubusercontent.com/psycopg/psycopg2/%APPVEYOR_REPO_COMMIT%/scripts/appveyor.py
+    - "%PYEXE% C:\\appveyor.py init"
+
+    # TODO: exporting variables below to be dropped once all the steps are
+    # moved to the appveyor.py script.
 
     # Set env variable according to the build environment
     - SET PYTHON=C:\Python%PYVER%
@@ -72,41 +75,11 @@ init:
     # Set Python to the path
     - SET PATH=%PYTHON%;%PYTHON%\Scripts;C:\Program Files\Git\mingw64\bin;%PATH%
 
-    # Verify Python version and architecture
-    - ECHO *******************************************************************
-    - ECHO Python Information
-    - ECHO *******************************************************************
-    - "%PYTHON%\\python --version"
-    - "%PYTHON%\\python -c \"import sys; print('64bit: ' + str(sys.maxsize > 2**32))\""
-
-    # Get & Install NASM
-    #- curl -L -o nasminst.exe https://www.nasm.us/pub/nasm/releasebuilds/2.12.02/win64/nasm-2.12.02-installer-x64.exe && start /wait nasminst.exe /S
-    #- SET PATH="C:\Program Files (x86)\nasm;%PATH%"
-
-    # Fix problem with VS2008 Express and 64bit builds
-    - ECHO Fixing VS2008 Express and 64bit builds
-    - COPY "C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\bin\\vcvars64.bat" "C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\bin\\amd64\\vcvarsamd64.bat"
-
-    # Fix problem with VS2010 Express 64bit missing vcvars64.bat
-    # Note: repository not cloned at this point, so need to fetch
-    # file another way
-    - ECHO Fixing VS2010 Express and 64bit builds
-    - curl -fsSL -o "C:\\Program Files (x86)\\Microsoft Visual Studio 10.0\\VC\\bin\\amd64\\vcvars64.bat" https://raw.githubusercontent.com/psycopg/psycopg2/master/scripts/vcvars64-vs2010.bat
-
     # Setup the compiler based upon version and architecture
     - ECHO Configuring Compiler
     - IF "%PYTHON_ARCH%"=="32" (CALL "C:\\Program Files (x86)\\Microsoft Visual Studio %VS_VER%\\VC\\vcvarsall.bat" x86)
     - IF "%PYTHON_ARCH%"=="64" (CALL "C:\\Program Files (x86)\\Microsoft Visual Studio %VS_VER%\\VC\\vcvarsall.bat" amd64)
 
-    # The program rc.exe on 64bit with some versions look in the wrong path
-    #   location when building postgresql.   This cheats by copying the x64 bit
-    #   files to that location.
-    - IF "%PYTHON_ARCH%"=="64" (COPY /Y "C:\\Program Files\\Microsoft SDKs\\Windows\\v7.0\\Bin\\x64\\rc*" "C:\\Program Files (x86)\\Microsoft SDKs\\Windows\\v7.0A\\Bin")
-
-    # Change PostgreSQL config before service starts to allow > 1 prepared
-    #   transactions for test cases
-    - ECHO max_prepared_transactions = 10 >> "C:\\Program Files\\PostgreSQL\\9.6\\data\\postgresql.conf"
-
 
 # Repository gets cloned, Cache is restored
 install:
@@ -236,3 +209,6 @@ test_script:
     - "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.__libpq_version__)\""
     - "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.extensions.libpq_version())\""
     - "%PYTHON%\\python.exe -c \"import tests; tests.unittest.main(defaultTest='tests.test_suite')\" --verbose"
+
+
+# vim: set ts=4 sts=4 sw=4:

scripts/appveyor.py

@@ -0,0 +1,277 @@
+#!/usr/bin/env python3
+"""
+Build steps for the windows binary packages.
+
+The script is designed to be called by appveyor. Subcommands map the steps in
+'appveyor.yml'.
+
+"""
+
+import re
+import os
+import sys
+import json
+import shutil
+import logging
+import subprocess as sp
+from glob import glob
+from urllib.request import urlopen
+from tempfile import NamedTemporaryFile
+from functools import lru_cache
+
+opt = None
+STEP_PREFIX = 'step_'
+
+logger = logging.getLogger()
+logging.basicConfig(
+    level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s'
+)
+
+
+def main():
+    global opt
+    opt = parse_cmdline()
+
+    setup_env()
+    cmd = globals()[STEP_PREFIX + opt.step]
+    cmd()
+
+
+@lru_cache()
+def setup_env():
+    """
+    Set the environment variables according to the build environment
+    """
+    python_info()
+
+    os.environ['VS_VER'] = vs_ver()
+
+    if vs_ver() == '10.0' and opt.arch_64:
+        os.environ['DISTUTILS_USE_SDK'] = '1'
+
+    os.environ['PATH'] = os.pathsep.join(
+        [
+            py_dir(),
+            os.path.join(py_dir(), 'Scripts'),
+            r'C:\Program Files\Git\mingw64\bin',
+            os.environ['PATH'],
+        ]
+    )
+
+    if vs_ver() == '9.0':
+        logger.info("Fixing VS2008 Express and 64bit builds")
+        shutil.copyfile(
+            os.path.join(vc_dir(), r"bin\vcvars64.bat"),
+            os.path.join(vc_dir(), r"bin\amd64\vcvarsamd64.bat"),
+        )
+
+    # Fix problem with VS2010 Express 64bit missing vcvars64.bat
+    # Note: repository not cloned at this point, so need to fetch
+    # file another way
+    if vs_ver() == '10.0':
+        if not os.path.exists(
+            os.path.join(vc_dir(), r"bin\amd64\vcvars64.bat")
+        ):
+            logger.info("Fixing VS2010 Express and 64bit builds")
+            with urlopen(
+                "https://raw.githubusercontent.com/psycopg/psycopg2/"
+                "master/scripts/vcvars64-vs2010.bat"
+            ) as f:
+                data = f.read()
+
+            with open(
+                os.path.join(vc_dir(), r"bin\amd64\vcvars64.bat"), 'w'
+            ) as f:
+                f.write(data)
+
+    logger.info("Configuring compiler")
+    bat_call(
+        [
+            os.path.join(vc_dir(), "vcvarsall.bat"),
+            'x86' if opt.arch_32 else 'amd64',
+        ]
+    )
+
+
+def python_info():
+    logger.info("Python Information")
+    out = call_command([py_exe(), '--version'], stderr=sp.STDOUT)
+    logger.info("%s", out)
+
+    cmdline = [
+        py_exe(),
+        '-c',
+        "import sys; print('64bit: %s' % (sys.maxsize > 2**32))",
+    ]
+    out = call_command(cmdline)
+    logger.info("%s", out)
+
+
+def step_init():
+    # The program rc.exe on 64bit with some versions look in the wrong path
+    # location when building postgresql. This cheats by copying the x64 bit
+    # files to that location.
+    if opt.arch_64:
+        for fn in glob(
+            r'C:\Program Files\Microsoft SDKs\Windows\v7.0\Bin\x64\rc*'
+        ):
+            logger.info("Copying %s to a better place" % os.path.basename(fn))
+            shutil.copy(
+                fn, r"C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Bin"
+            )
+
+    # Change PostgreSQL config before service starts
+    logger.info("Configuring Postgres")
+    with open(os.path.join(pg_dir(), 'data', 'postgresql.conf'), 'a') as f:
+        # allow > 1 prepared transactions for test cases
+        print("max_prepared_transactions = 10", file=f)
+
+
+def bat_call(cmdline):
+    """
+    Simulate 'CALL' from a batch file
+
+    Execute CALL *cmdline* and export the changed environment to the current
+    environment.
+
+    nana-nana-nana-nana...
+
+    """
+    if not isinstance(cmdline, str):
+        cmdline = ' '.join(c if ' ' not in c else '"%s"' % c for c in cmdline)
+
+    pyexe = py_exe()
+
+    data = f"""\
+CALL {cmdline}
+{pyexe} -c "import os, sys, json; json.dump(dict(os.environ), sys.stdout, indent=2)"
+"""
+
+    logger.debug("preparing file to batcall:\n\n%s", data)
+
+    with NamedTemporaryFile(suffix='.bat') as tmp:
+        fn = tmp.name
+
+    with open(fn, "w") as f:
+        f.write(data)
+
+    try:
+        out = call_command(fn)
+        # be vewwy vewwy caweful to print the env var as it might contain
+        # secwet things like your pwecious pwivate key.
+        # logger.debug("output of command:\n\n%s", out.decode('utf8', 'replace'))
+
+        # The output has some useless crap on stdout, because sure, and json
+        # indented so the last { on column 1 is where we have to start parsing
+
+        m = list(re.finditer(b'^{', out, re.MULTILINE))[-1]
+        out = out[m.start() :]
+        env = json.loads(out)
+        for k, v in env.items():
+            if os.environ.get(k) != v:
+                logger.info("setting %s=%s", k, v)
+                os.environ[k] = v
+    finally:
+        os.remove(fn)
+
+
+def py_dir():
+    """
+    Return the path to the target python binary to execute.
+    """
+    dirname = ''.join(
+        [r"C:\Python", opt.pyver, '-x64' if opt.arch_64 else '']
+    )
+    return dirname
+
+
+def py_exe():
+    """
+    Return the full path of the target python executable.
+    """
+    return os.path.join(py_dir(), 'python.exe')
+
+
+def vc_dir(vsver=None):
+    """
+    Return the path of the Visual C compiler.
+    """
+    if vsver is None:
+        vsver = vs_ver()
+
+    return r"C:\Program Files (x86)\Microsoft Visual Studio %s\VC" % vsver
+
+
+def vs_ver(pyver=None):
+    # Py 2.7 = VS Ver. 9.0 (VS 2008)
+    # Py 3.4 = VS Ver. 10.0 (VS 2010)
+    # Py 3.5, 3.6, 3.7 = VS Ver. 14.0 (VS 2015)
+    if pyver is None:
+        pyver = opt.pyver
+
+    if pyver == '27':
+        vsver = '9.0'
+    elif pyver == '34':
+        vsver = '10.0'
+    elif pyver in ('35', '36', '37'):
+        vsver = '14.0'
+    else:
+        raise Exception('unexpected python version: %r' % pyver)
+
+    return vsver
+
+
+def pg_dir():
+    return r"C:\Program Files\PostgreSQL\9.6"
+
+
+def call_command(cmdline, **kwargs):
+    logger.debug("calling command: %s", cmdline)
+    data = sp.check_output(cmdline, **kwargs)
+    return data
+
+
+def parse_cmdline():
+    from argparse import ArgumentParser
+
+    parser = ArgumentParser(description=__doc__)
+
+    parser.add_argument(
+        '--pyver',
+        choices='27 34 35 36 37'.split(),
+        help="the target python version. Default from PYVER env var",
+    )
+
+    parser.add_argument(
+        '--pyarch',
+        choices='32 64'.split(),
+        help="the target python architecture. Default from PYTHON_ARCH env var",
+    )
+
+    steps = [
+        n[len(STEP_PREFIX) :]
+        for n in globals()
+        if n.startswith(STEP_PREFIX) and callable(globals()[n])
+    ]
+
+    parser.add_argument(
+        'step', choices=steps, help="the appveyor step to execute"
+    )
+
+    opt = parser.parse_args()
+
+    # And die if they are not there.
+    if not opt.pyver:
+        opt.pyver = os.environ['PYVER']
+    if not opt.pyarch:
+        opt.pyarch = os.environ['PYTHON_ARCH']
+        assert opt.pyarch in ('32', '64')
+
+    opt.arch_32 = opt.pyarch == '32'
+    opt.arch_64 = opt.pyarch == '64'
+
+    return opt
+
+
+if __name__ == '__main__':
+    sys.exit(main())