From 00fc2820a008adaea33e2fa6fe6e64d7d8e9e29a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 14 Apr 2019 20:10:51 +0100 Subject: [PATCH 01/17] Added script to implement appveyor functionality in Python Only the init step for the moment. --- .appveyor.yml | 48 ++------ scripts/appveyor.py | 277 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+), 36 deletions(-) create mode 100755 scripts/appveyor.py diff --git a/.appveyor.yml b/.appveyor.yml index 81515e40..307e101e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,8 +6,6 @@ environment: global: # MSVC Express 2008's setenv.cmd failes if /E:ON and /V:ON are not # enabled in the batch script interpreter - # - # WITH_COMPILER: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_compiler.cmd" CMD_IN_ENV: cmd /E:ON /V:ON /C .\appveyor\run_with_env.cmd matrix: @@ -29,13 +27,14 @@ environment: PSYCOPG2_TESTDB: psycopg2_test PSYCOPG2_TESTDB_USER: postgres - PSYCOPG2_TESTDB_PASSWORD: Password12! PSYCOPG2_TESTDB_HOST: localhost - PSYCOPG2_TESTDB_PORT: 5432 PGUSER: postgres PGPASSWORD: Password12! + # The python used in the build process, not the one packages are built for + PYEXE: C:\Python36\python.exe + matrix: fast_finish: false @@ -51,8 +50,12 @@ cache: # Script called before repo cloning init: - # Uncomment next line to get RDP access during the build. - #- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + # TODO: move functionalities out of init to drop this fetch problem + - curl -fsSL -o "C:\\appveyor.py" https://raw.githubusercontent.com/psycopg/psycopg2/%APPVEYOR_REPO_COMMIT%/scripts/appveyor.py + - "%PYEXE% C:\\appveyor.py init" + + # TODO: exporting variables below to be dropped once all the steps are + # moved to the appveyor.py script. # Set env variable according to the build environment - SET PYTHON=C:\Python%PYVER% @@ -72,41 +75,11 @@ init: # Set Python to the path - SET PATH=%PYTHON%;%PYTHON%\Scripts;C:\Program Files\Git\mingw64\bin;%PATH% - # Verify Python version and architecture - - ECHO ******************************************************************* - - ECHO Python Information - - ECHO ******************************************************************* - - "%PYTHON%\\python --version" - - "%PYTHON%\\python -c \"import sys; print('64bit: ' + str(sys.maxsize > 2**32))\"" - - # Get & Install NASM - #- curl -L -o nasminst.exe https://www.nasm.us/pub/nasm/releasebuilds/2.12.02/win64/nasm-2.12.02-installer-x64.exe && start /wait nasminst.exe /S - #- SET PATH="C:\Program Files (x86)\nasm;%PATH%" - - # Fix problem with VS2008 Express and 64bit builds - - ECHO Fixing VS2008 Express and 64bit builds - - COPY "C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\bin\\vcvars64.bat" "C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\bin\\amd64\\vcvarsamd64.bat" - - # Fix problem with VS2010 Express 64bit missing vcvars64.bat - # Note: repository not cloned at this point, so need to fetch - # file another way - - ECHO Fixing VS2010 Express and 64bit builds - - curl -fsSL -o "C:\\Program Files (x86)\\Microsoft Visual Studio 10.0\\VC\\bin\\amd64\\vcvars64.bat" https://raw.githubusercontent.com/psycopg/psycopg2/master/scripts/vcvars64-vs2010.bat - # Setup the compiler based upon version and architecture - ECHO Configuring Compiler - IF "%PYTHON_ARCH%"=="32" (CALL "C:\\Program Files (x86)\\Microsoft Visual Studio %VS_VER%\\VC\\vcvarsall.bat" x86) - IF "%PYTHON_ARCH%"=="64" (CALL "C:\\Program Files (x86)\\Microsoft Visual Studio %VS_VER%\\VC\\vcvarsall.bat" amd64) - # The program rc.exe on 64bit with some versions look in the wrong path - # location when building postgresql. This cheats by copying the x64 bit - # files to that location. - - IF "%PYTHON_ARCH%"=="64" (COPY /Y "C:\\Program Files\\Microsoft SDKs\\Windows\\v7.0\\Bin\\x64\\rc*" "C:\\Program Files (x86)\\Microsoft SDKs\\Windows\\v7.0A\\Bin") - - # Change PostgreSQL config before service starts to allow > 1 prepared - # transactions for test cases - - ECHO max_prepared_transactions = 10 >> "C:\\Program Files\\PostgreSQL\\9.6\\data\\postgresql.conf" - # Repository gets cloned, Cache is restored install: @@ -236,3 +209,6 @@ test_script: - "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.__libpq_version__)\"" - "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.extensions.libpq_version())\"" - "%PYTHON%\\python.exe -c \"import tests; tests.unittest.main(defaultTest='tests.test_suite')\" --verbose" + + +# vim: set ts=4 sts=4 sw=4: diff --git a/scripts/appveyor.py b/scripts/appveyor.py new file mode 100755 index 00000000..0fcaa154 --- /dev/null +++ b/scripts/appveyor.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +""" +Build steps for the windows binary packages. + +The script is designed to be called by appveyor. Subcommands map the steps in +'appveyor.yml'. + +""" + +import re +import os +import sys +import json +import shutil +import logging +import subprocess as sp +from glob import glob +from urllib.request import urlopen +from tempfile import NamedTemporaryFile +from functools import lru_cache + +opt = None +STEP_PREFIX = 'step_' + +logger = logging.getLogger() +logging.basicConfig( + level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s' +) + + +def main(): + global opt + opt = parse_cmdline() + + setup_env() + cmd = globals()[STEP_PREFIX + opt.step] + cmd() + + +@lru_cache() +def setup_env(): + """ + Set the environment variables according to the build environment + """ + python_info() + + os.environ['VS_VER'] = vs_ver() + + if vs_ver() == '10.0' and opt.arch_64: + os.environ['DISTUTILS_USE_SDK'] = '1' + + os.environ['PATH'] = os.pathsep.join( + [ + py_dir(), + os.path.join(py_dir(), 'Scripts'), + r'C:\Program Files\Git\mingw64\bin', + os.environ['PATH'], + ] + ) + + if vs_ver() == '9.0': + logger.info("Fixing VS2008 Express and 64bit builds") + shutil.copyfile( + os.path.join(vc_dir(), r"bin\vcvars64.bat"), + os.path.join(vc_dir(), r"bin\amd64\vcvarsamd64.bat"), + ) + + # Fix problem with VS2010 Express 64bit missing vcvars64.bat + # Note: repository not cloned at this point, so need to fetch + # file another way + if vs_ver() == '10.0': + if not os.path.exists( + os.path.join(vc_dir(), r"bin\amd64\vcvars64.bat") + ): + logger.info("Fixing VS2010 Express and 64bit builds") + with urlopen( + "https://raw.githubusercontent.com/psycopg/psycopg2/" + "master/scripts/vcvars64-vs2010.bat" + ) as f: + data = f.read() + + with open( + os.path.join(vc_dir(), r"bin\amd64\vcvars64.bat"), 'w' + ) as f: + f.write(data) + + logger.info("Configuring compiler") + bat_call( + [ + os.path.join(vc_dir(), "vcvarsall.bat"), + 'x86' if opt.arch_32 else 'amd64', + ] + ) + + +def python_info(): + logger.info("Python Information") + out = call_command([py_exe(), '--version'], stderr=sp.STDOUT) + logger.info("%s", out) + + cmdline = [ + py_exe(), + '-c', + "import sys; print('64bit: %s' % (sys.maxsize > 2**32))", + ] + out = call_command(cmdline) + logger.info("%s", out) + + +def step_init(): + # The program rc.exe on 64bit with some versions look in the wrong path + # location when building postgresql. This cheats by copying the x64 bit + # files to that location. + if opt.arch_64: + for fn in glob( + r'C:\Program Files\Microsoft SDKs\Windows\v7.0\Bin\x64\rc*' + ): + logger.info("Copying %s to a better place" % os.path.basename(fn)) + shutil.copy( + fn, r"C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Bin" + ) + + # Change PostgreSQL config before service starts + logger.info("Configuring Postgres") + with open(os.path.join(pg_dir(), 'data', 'postgresql.conf'), 'a') as f: + # allow > 1 prepared transactions for test cases + print("max_prepared_transactions = 10", file=f) + + +def bat_call(cmdline): + """ + Simulate 'CALL' from a batch file + + Execute CALL *cmdline* and export the changed environment to the current + environment. + + nana-nana-nana-nana... + + """ + if not isinstance(cmdline, str): + cmdline = ' '.join(c if ' ' not in c else '"%s"' % c for c in cmdline) + + pyexe = py_exe() + + data = f"""\ +CALL {cmdline} +{pyexe} -c "import os, sys, json; json.dump(dict(os.environ), sys.stdout, indent=2)" +""" + + logger.debug("preparing file to batcall:\n\n%s", data) + + with NamedTemporaryFile(suffix='.bat') as tmp: + fn = tmp.name + + with open(fn, "w") as f: + f.write(data) + + try: + out = call_command(fn) + # be vewwy vewwy caweful to print the env var as it might contain + # secwet things like your pwecious pwivate key. + # logger.debug("output of command:\n\n%s", out.decode('utf8', 'replace')) + + # The output has some useless crap on stdout, because sure, and json + # indented so the last { on column 1 is where we have to start parsing + + m = list(re.finditer(b'^{', out, re.MULTILINE))[-1] + out = out[m.start() :] + env = json.loads(out) + for k, v in env.items(): + if os.environ.get(k) != v: + logger.info("setting %s=%s", k, v) + os.environ[k] = v + finally: + os.remove(fn) + + +def py_dir(): + """ + Return the path to the target python binary to execute. + """ + dirname = ''.join( + [r"C:\Python", opt.pyver, '-x64' if opt.arch_64 else ''] + ) + return dirname + + +def py_exe(): + """ + Return the full path of the target python executable. + """ + return os.path.join(py_dir(), 'python.exe') + + +def vc_dir(vsver=None): + """ + Return the path of the Visual C compiler. + """ + if vsver is None: + vsver = vs_ver() + + return r"C:\Program Files (x86)\Microsoft Visual Studio %s\VC" % vsver + + +def vs_ver(pyver=None): + # Py 2.7 = VS Ver. 9.0 (VS 2008) + # Py 3.4 = VS Ver. 10.0 (VS 2010) + # Py 3.5, 3.6, 3.7 = VS Ver. 14.0 (VS 2015) + if pyver is None: + pyver = opt.pyver + + if pyver == '27': + vsver = '9.0' + elif pyver == '34': + vsver = '10.0' + elif pyver in ('35', '36', '37'): + vsver = '14.0' + else: + raise Exception('unexpected python version: %r' % pyver) + + return vsver + + +def pg_dir(): + return r"C:\Program Files\PostgreSQL\9.6" + + +def call_command(cmdline, **kwargs): + logger.debug("calling command: %s", cmdline) + data = sp.check_output(cmdline, **kwargs) + return data + + +def parse_cmdline(): + from argparse import ArgumentParser + + parser = ArgumentParser(description=__doc__) + + parser.add_argument( + '--pyver', + choices='27 34 35 36 37'.split(), + help="the target python version. Default from PYVER env var", + ) + + parser.add_argument( + '--pyarch', + choices='32 64'.split(), + help="the target python architecture. Default from PYTHON_ARCH env var", + ) + + steps = [ + n[len(STEP_PREFIX) :] + for n in globals() + if n.startswith(STEP_PREFIX) and callable(globals()[n]) + ] + + parser.add_argument( + 'step', choices=steps, help="the appveyor step to execute" + ) + + opt = parser.parse_args() + + # And die if they are not there. + if not opt.pyver: + opt.pyver = os.environ['PYVER'] + if not opt.pyarch: + opt.pyarch = os.environ['PYTHON_ARCH'] + assert opt.pyarch in ('32', '64') + + opt.arch_32 = opt.pyarch == '32' + opt.arch_64 = opt.pyarch == '64' + + return opt + + +if __name__ == '__main__': + sys.exit(main()) From 37ce131d2c53a57e6fcdbc524bf57eef28244773 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 14 Apr 2019 22:02:52 +0100 Subject: [PATCH 02/17] Added setenv to log env vars changes --- scripts/appveyor.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/scripts/appveyor.py b/scripts/appveyor.py index 0fcaa154..d7ea02f0 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -44,19 +44,18 @@ def setup_env(): """ python_info() - os.environ['VS_VER'] = vs_ver() + setenv('VS_VER', vs_ver()) if vs_ver() == '10.0' and opt.arch_64: - os.environ['DISTUTILS_USE_SDK'] = '1' + setenv('DISTUTILS_USE_SDK', '1') - os.environ['PATH'] = os.pathsep.join( - [ - py_dir(), - os.path.join(py_dir(), 'Scripts'), - r'C:\Program Files\Git\mingw64\bin', - os.environ['PATH'], - ] - ) + path = [ + py_dir(), + os.path.join(py_dir(), 'Scripts'), + r'C:\Program Files\Git\mingw64\bin', + os.environ['PATH'], + ] + setenv('PATH', os.pathsep.join(path)) if vs_ver() == '9.0': logger.info("Fixing VS2008 Express and 64bit builds") @@ -169,8 +168,7 @@ CALL {cmdline} env = json.loads(out) for k, v in env.items(): if os.environ.get(k) != v: - logger.info("setting %s=%s", k, v) - os.environ[k] = v + setenv(k, v) finally: os.remove(fn) @@ -179,9 +177,7 @@ def py_dir(): """ Return the path to the target python binary to execute. """ - dirname = ''.join( - [r"C:\Python", opt.pyver, '-x64' if opt.arch_64 else ''] - ) + dirname = ''.join([r"C:\Python", opt.pyver, '-x64' if opt.arch_64 else '']) return dirname @@ -231,6 +227,11 @@ def call_command(cmdline, **kwargs): return data +def setenv(k, v): + logger.info("setting %s=%s", k, v) + os.environ[k] = v + + def parse_cmdline(): from argparse import ArgumentParser From 73f6a0cd9532a6e4b9775595457324722988e8c0 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 14 Apr 2019 22:43:28 +0100 Subject: [PATCH 03/17] Build openssl from Python --- .appveyor.yml | 46 +-------------------- scripts/appveyor.py | 97 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 94 insertions(+), 49 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 307e101e..6de23903 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -83,51 +83,7 @@ init: # Repository gets cloned, Cache is restored install: - # We start off CD'ed to cloned folder - - SET BASE_DIR=C:\Others\%PYTHON_ARCH%\%VS_VER% - - SET BUILD_DIR=%BASE_DIR%\Builds - - IF NOT EXIST %BUILD_DIR% MKDIR %BUILD_DIR% - - - ECHO ******************************************************************* - - ECHO Initialized variables specific for this build - - ECHO ******************************************************************* - - ECHO %BASE_DIR% - - ECHO %BUILD_DIR% - - ECHO ******************************************************************* - - # Setup directories for building OpenSSL libraries - - ECHO ******************************************************************* - - ECHO Preparing for building OpenSSL - - ECHO ******************************************************************* - - SET OPENSSLTOP=%BASE_DIR%\openssl - - IF NOT EXIST %OPENSSLTOP%\include\openssl MKDIR %OPENSSLTOP%\include\openssl - - IF NOT EXIST %OPENSSLTOP%\lib MKDIR %OPENSSLTOP%\lib - - # Setup OpenSSL Environment Variables based on processor architecture - - ps: >- - If ($env:PYTHON_ARCH -Match "32" ) { - $env:VCVARS_PLATFORM="x86" - $env:TARGET="VC-WIN32" - } Else { - $env:VCVARS_PLATFORM="amd64" - $env:TARGET="VC-WIN64A" - $env:CPU="AMD64" - } - # Download OpenSSL source - - CD C:\Others - - IF NOT EXIST OpenSSL_%OPENSSL_VERSION%.zip ( - curl -fsSL -o OpenSSL_%OPENSSL_VERSION%.zip https://github.com/openssl/openssl/archive/OpenSSL_%OPENSSL_VERSION%.zip - ) - - - IF NOT EXIST %OPENSSLTOP%\lib\libssl.lib ( - CD %BUILD_DIR% && - 7z x C:\Others\OpenSSL_%OPENSSL_VERSION%.zip && - CD openssl-OpenSSL_%OPENSSL_VERSION% && - perl Configure %TARGET% no-asm no-shared no-zlib --prefix=%OPENSSLTOP% --openssldir=%OPENSSLTOP% && - nmake build_libs install_dev && - CD %BASE_DIR% && - RMDIR /S /Q %BUILD_DIR%\openssl-OpenSSL_%OPENSSL_VERSION% - ) + - "%PYEXE% C:\\appveyor.py install" # Setup directories for building PostgreSQL librarires - ECHO ******************************************************************* diff --git a/scripts/appveyor.py b/scripts/appveyor.py index d7ea02f0..688519a3 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -15,9 +15,10 @@ import shutil import logging import subprocess as sp from glob import glob -from urllib.request import urlopen +from zipfile import ZipFile from tempfile import NamedTemporaryFile from functools import lru_cache +from urllib.request import urlopen opt = None STEP_PREFIX = 'step_' @@ -126,6 +127,73 @@ def step_init(): print("max_prepared_transactions = 10", file=f) +def step_install(): + build_openssl() + + +def build_openssl(): + # Setup directories for building OpenSSL libraries + top = os.path.join(base_dir(), 'openssl') + ensure_dir(os.path.join(top, 'include', 'openssl')) + ensure_dir(os.path.join(top, 'lib')) + + # Setup OpenSSL Environment Variables based on processor architecture + if opt.arch_32: + target = 'VC-WIN32' + setenv('VCVARS_PLATFORM', 'x86') + else: + target = 'VC-WIN64A' + setenv('VCVARS_PLATFORM', 'amd64') + setenv('CPU', 'AMD64') + + ver = os.environ['OPENSSL_VERSION'] + + # Download OpenSSL source + zipname = f'OpenSSL_{ver}.zip' + zipfile = os.path.join(r'C:\Others', zipname) + if not os.path.exists(zipfile): + download( + f"https://github.com/openssl/openssl/archive/{zipname}", zipfile + ) + + if os.path.exists(os.path.join(top, 'lib', 'libssl.lib')): + return + + with ZipFile(zipfile) as z: + z.extractall(path=build_dir()) + + os.chdir(os.path.join(build_dir(), f"openssl-OpenSSL_{ver}")) + cmdline = [ + 'perl', + 'Configure', + target, + 'no-asm', + 'no-shared', + 'no-zlib', + f'--prefix={top}', + f'--openssldir={top}', + ] + call_command(cmdline, output=False) + + cmdline = "nmake build_libs install_dev".split() + call_command(cmdline, output=False) + + assert os.path.exists(os.path.join(top, 'lib', 'libssl.lib')) + + os.chdir(base_dir()) + shutil.rmtree(os.path.join(build_dir(), f"openssl-OpenSSL_{ver}")) + + +def download(url, fn): + """Download a file locally""" + with open(fn, 'wb') as fo, urlopen(url) as fi: + while 1: + data = fi.read(8192) + if not data: + break + fo.write(data) + + def bat_call(cmdline): """ Simulate 'CALL' from a batch file @@ -221,10 +289,31 @@ def pg_dir(): return r"C:\Program Files\PostgreSQL\9.6" -def call_command(cmdline, **kwargs): +def base_dir(): + rv = r"C:\Others\%s\%s" % (opt.pyarch, vs_ver()) + return ensure_dir(rv) + + +def build_dir(): + rv = os.path.join(base_dir(), 'Builds') + return ensure_dir(rv) + + +def ensure_dir(dir): + if not os.path.exists(dir): + logger.info("creating directory %s", dir) + os.makedirs(dir) + + return dir + + +def call_command(cmdline, output=True, **kwargs): logger.debug("calling command: %s", cmdline) - data = sp.check_output(cmdline, **kwargs) - return data + if output: + data = sp.check_output(cmdline, **kwargs) + return data + else: + sp.check_call(cmdline, **kwargs) def setenv(k, v): From c87519743222e9579438504c729086c827a4df42 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 15 Apr 2019 00:38:51 +0100 Subject: [PATCH 04/17] Build libpq from Python --- .appveyor.yml | 54 --------- scripts/appveyor.py | 219 +++++++++++++++++++++++++++++------ scripts/win_openssl_11.patch | 20 ---- 3 files changed, 186 insertions(+), 107 deletions(-) delete mode 100644 scripts/win_openssl_11.patch diff --git a/.appveyor.yml b/.appveyor.yml index 6de23903..8ad507e5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -85,60 +85,6 @@ init: install: - "%PYEXE% C:\\appveyor.py install" - # Setup directories for building PostgreSQL librarires - - ECHO ******************************************************************* - - ECHO Preparing for building PostgreSQL libraries - - ECHO ******************************************************************* - - SET PGTOP=%BASE_DIR%\postgresql - - IF NOT EXIST %PGTOP%\include MKDIR %PGTOP%\include - - IF NOT EXIST %PGTOP%\lib MKDIR %PGTOP%\lib - - IF NOT EXIST %PGTOP%\bin MKDIR %PGTOP%\bin - - # Download PostgreSQL source - - CD C:\Others - - IF NOT EXIST postgres-REL_%POSTGRES_VERSION%.zip ( - curl -fsSL -o postgres-REL_%POSTGRES_VERSION%.zip https://github.com/postgres/postgres/archive/REL_%POSTGRES_VERSION%.zip - ) - - # Setup build config file (config.pl) - # Hack the Mkvcbuild.pm file so we build the lib version of libpq - # Build libpgport, libpgcommon, libpq - # Install includes - # Copy over built libraries - # Prepare local include directory for building from - # Build pg_config in place - # Note patch for OpenSSL 1.1 configuration. See: - # https://www.postgresql-archive.org/Compile-psql-9-6-with-SSL-Version-1-1-0-td6054118.html - # NOTE: Cannot set and use the same variable inside an IF - - SET PGBUILD=%BUILD_DIR%\postgres-REL_%POSTGRES_VERSION% - - IF NOT EXIST %PGTOP%\lib\libpq.lib ( - CD %BUILD_DIR% && - 7z x C:\Others\postgres-REL_%POSTGRES_VERSION%.zip && - CD postgres-REL_%POSTGRES_VERSION% && - patch -p1 < %APPVEYOR_BUILD_FOLDER%\scripts\win_openssl_11.patch && - CD src\tools\msvc && - ECHO $config-^>{ldap} = 0; > config.pl && - ECHO $config-^>{openssl} = "%OPENSSLTOP:\=\\%"; >> config.pl && - ECHO.>> config.pl && - ECHO 1;>> config.pl && - perl -pi.bak -e "s/'libpq', 'dll'/'libpq', 'lib'/g" Mkvcbuild.pm && - build libpgport && - build libpgcommon && - build libpq && - ECHO "" > %PGBUILD%\src\backend\parser\gram.h && - perl -pi.bak -e "s/qw\(Install\)/qw\(Install CopyIncludeFiles\)/g" Install.pm && - perl -MInstall=CopyIncludeFiles -e"chdir('../../..'); CopyIncludeFiles('%PGTOP%')" && - COPY %PGBUILD%\Release\libpgport\libpgport.lib %PGTOP%\lib && - COPY %PGBUILD%\Release\libpgcommon\libpgcommon.lib %PGTOP%\lib && - COPY %PGBUILD%\Release\libpq\libpq.lib %PGTOP%\lib && - XCOPY /Y /S %PGBUILD%\src\include\port\win32\* %PGBUILD%\src\include && - XCOPY /Y /S %PGBUILD%\src\include\port\win32_msvc\* %PGBUILD%\src\include && - CD %PGBUILD%\src\bin\pg_config && - cl pg_config.c /MT /nologo /I%PGBUILD%\src\include /link /LIBPATH:%PGTOP%\lib libpgcommon.lib libpgport.lib advapi32.lib /NODEFAULTLIB:libcmt.lib /OUT:%PGTOP%\bin\pg_config.exe && - CD %BASE_DIR% && - RMDIR /S /Q %PGBUILD% - ) - build: off #before_build: diff --git a/scripts/appveyor.py b/scripts/appveyor.py index 688519a3..4ed758d0 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -95,15 +95,13 @@ def setup_env(): def python_info(): logger.info("Python Information") - out = call_command([py_exe(), '--version'], stderr=sp.STDOUT) + out = out_command([py_exe(), '--version'], stderr=sp.STDOUT) logger.info("%s", out) - cmdline = [ - py_exe(), - '-c', - "import sys; print('64bit: %s' % (sys.maxsize > 2**32))", - ] - out = call_command(cmdline) + out = out_command( + [py_exe(), '-c'] + + ["import sys; print('64bit: %s' % (sys.maxsize > 2**32))"] + ) logger.info("%s", out) @@ -128,12 +126,19 @@ def step_init(): def step_install(): - build_openssl() + # TODO: enable again + # build_openssl() + build_libpq() def build_openssl(): - # Setup directories for building OpenSSL libraries top = os.path.join(base_dir(), 'openssl') + if os.path.exists(os.path.join(top, 'lib', 'libssl.lib')): + return + + logger.info("Building OpenSSL") + + # Setup directories for building OpenSSL libraries ensure_dir(os.path.join(top, 'include', 'openssl')) ensure_dir(os.path.join(top, 'lib')) @@ -156,27 +161,16 @@ def build_openssl(): f"https://github.com/openssl/openssl/archive/{zipname}", zipfile ) - if os.path.exists(os.path.join(top, 'lib', 'libssl.lib')): - return - with ZipFile(zipfile) as z: z.extractall(path=build_dir()) os.chdir(os.path.join(build_dir(), f"openssl-OpenSSL_{ver}")) - cmdline = [ - 'perl', - 'Configure', - target, - 'no-asm', - 'no-shared', - 'no-zlib', - f'--prefix={top}', - f'--openssldir={top}', - ] - call_command(cmdline, output=False) + run_command( + ['perl', 'Configure', target, 'no-asm'] + + ['no-shared', 'no-zlib', f'--prefix={top}', f'--openssldir={top}'] + ) - cmdline = "nmake build_libs install_dev".split() - call_command(cmdline, output=False) + run_command("nmake build_libs install_dev".split()) assert os.path.exists(os.path.join(top, 'lib', 'libssl.lib')) @@ -184,8 +178,115 @@ def build_openssl(): shutil.rmtree(os.path.join(build_dir(), f"openssl-OpenSSL_{ver}")) +def build_libpq(): + top = os.path.join(base_dir(), 'postgresql') + if os.path.exists(os.path.join(top, 'lib', 'libpq.lib')): + return + + logger.info("Building libpq") + + # Setup directories for building PostgreSQL librarires + ensure_dir(os.path.join(top, 'include')) + ensure_dir(os.path.join(top, 'lib')) + ensure_dir(os.path.join(top, 'bin')) + + ver = os.environ['POSTGRES_VERSION'] + + # Download PostgreSQL source + zipname = f'postgres-REL_{ver}.zip' + zipfile = os.path.join(r'C:\Others', zipname) + if not os.path.exists(zipfile): + download( + f"https://github.com/postgres/postgres/archive/REL_{ver}.zip", + zipfile, + ) + + with ZipFile(zipfile) as z: + z.extractall(path=build_dir()) + + pgbuild = os.path.join(build_dir(), f"postgres-REL_{ver}") + os.chdir(pgbuild) + + # Patch for OpenSSL 1.1 configuration. See: + # https://www.postgresql-archive.org/Compile-psql-9-6-with-SSL-Version-1-1-0-td6054118.html + assert os.path.exists("src/include/pg_config.h.win32") + with open("src/include/pg_config.h.win32", 'a') as f: + print( + """ +#define HAVE_ASN1_STRING_GET0_DATA 1 +#define HAVE_BIO_GET_DATA 1 +#define HAVE_BIO_METH_NEW 1 +#define HAVE_OPENSSL_INIT_SSL 1 +""", + file=f, + ) + + # Setup build config file (config.pl) + os.chdir("src/tools/msvc") + with open("config.pl", 'w') as f: + print( + """\ +$config->{ldap} = 0; +$config->{openssl} = "%s"; + +1; +""" + % os.path.join(base_dir(), 'openssl').replace('\\', '\\\\'), + file=f, + ) + + # Hack the Mkvcbuild.pm file so we build the lib version of libpq + file_replace('Mkvcbuild.pm', "'libpq', 'dll'", "'libpq', 'lib'") + + # Build libpgport, libpgcommon, libpq + run_command([which("build"), "libpgport"]) + run_command([which("build"), "libpgcommon"]) + run_command([which("build"), "libpq"]) + + # Install includes + with open(os.path.join(pgbuild, "src/backend/parser/gram.h"), "w") as f: + print("", file=f) + + # Copy over built libraries + file_replace("Install.pm", "qw(Install)", "qw(Install CopyIncludeFiles)") + run_command( + ["perl", "-MInstall=CopyIncludeFiles", "-e"] + + [f"chdir('../../..'); CopyIncludeFiles('{top}')"] + ) + + for lib in ('libpgport', 'libpgcommon', 'libpq'): + shutil.copy( + os.path.join(pgbuild, f'Release/{lib}/{lib}.lib'), + os.path.join(top, 'lib'), + ) + + # Prepare local include directory for building from + for dir in ('win32', 'win32_msvc'): + merge_dir( + os.path.join(pgbuild, f"src/include/port/{dir}"), + os.path.join(pgbuild, "src/include"), + ) + + # Build pg_config in place + os.chdir(os.path.join(pgbuild, 'src/bin/pg_config')) + run_command( + ['cl', 'pg_config.c', '/MT', '/nologo', fr'/I{pgbuild}\src\include'] + + ['/link', fr'/LIBPATH:{top}\lib'] + + ['libpgcommon.lib', 'libpgport.lib', 'advapi32.lib'] + + ['/NODEFAULTLIB:libcmt.lib'] + + [fr'/OUT:{top}\bin\pg_config.exe'] + ) + + assert os.path.exists(os.path.join(top, 'lib', 'libpq.lib')) + assert os.path.exists(os.path.join(top, 'bin', 'pg_config.exe')) + + os.chdir(base_dir()) + shutil.rmtree(os.path.join(pgbuild)) + + def download(url, fn): """Download a file locally""" + logger.info("downloading %s", url) with open(fn, 'wb') as fo, urlopen(url) as fi: while 1: data = fi.read(8192) @@ -193,6 +294,37 @@ def download(url, fn): break fo.write(data) + logger.info("file downloaded: %s", fn) + + +def file_replace(fn, s1, s2): + """ + Replace all the occurrences of the string s1 into s2 in the file fn. + """ + assert os.path.exists(fn) + with open(fn, 'r+') as f: + data = f.read() + f.seek(0) + f.write(data.replace(s1, s2)) + f.truncate() + + +def merge_dir(src, tgt): + """ + Merge the content of the directory src into the directory tgt + + Reproduce the semantic of "XCOPY /Y /S src/* tgt" + """ + for dp, _dns, fns in os.walk(src): + logger.debug("dirpath %s", dp) + if not fns: + continue + assert dp.startswith(src) + subdir = dp[len(src) :].lstrip(os.sep) + tgtdir = ensure_dir(os.path.join(tgt, subdir)) + for fn in fns: + shutil.copy(os.path.join(dp, fn), tgtdir) + def bat_call(cmdline): """ @@ -223,7 +355,7 @@ CALL {cmdline} f.write(data) try: - out = call_command(fn) + out = out_command(fn) # be vewwy vewwy caweful to print the env var as it might contain # secwet things like your pwecious pwivate key. # logger.debug("output of command:\n\n%s", out.decode('utf8', 'replace')) @@ -307,20 +439,41 @@ def ensure_dir(dir): return dir -def call_command(cmdline, output=True, **kwargs): +def run_command(cmdline, **kwargs): logger.debug("calling command: %s", cmdline) - if output: - data = sp.check_output(cmdline, **kwargs) - return data - else: - sp.check_call(cmdline, **kwargs) + sp.check_call(cmdline, **kwargs) + + +def out_command(cmdline, **kwargs): + logger.debug("calling command: %s", cmdline) + data = sp.check_output(cmdline, **kwargs) + return data def setenv(k, v): - logger.info("setting %s=%s", k, v) + logger.debug("setting %s=%s", k, v) os.environ[k] = v +def which(name): + """ + Return the full path of a command found on the path + """ + base, ext = os.path.splitext(name) + if not ext: + exts = ('.com', '.exe', '.bat', '.cmd') + else: + exts = (ext,) + + for dir in ['.'] + os.environ['PATH'].split(os.pathsep): + for ext in exts: + fn = os.path.join(dir, base + ext) + if os.path.isfile(fn): + return fn + + raise Exception("couldn't find program on path: %s" % name) + + def parse_cmdline(): from argparse import ArgumentParser diff --git a/scripts/win_openssl_11.patch b/scripts/win_openssl_11.patch deleted file mode 100644 index 57d0bd81..00000000 --- a/scripts/win_openssl_11.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/src/include/pg_config.h.win32 b/src/include/pg_config.h.win32 -index dfd6972383..72d1259ffb 100644 ---- a/src/include/pg_config.h.win32 -+++ b/src/include/pg_config.h.win32 -@@ -74,12 +74,15 @@ - - /* Define to 1 if you have the `ASN1_STRING_get0_data' function. */ - /* #undef HAVE_ASN1_STRING_GET0_DATA */ -+#define HAVE_ASN1_STRING_GET0_DATA 1 - - /* Define to 1 if you have the `BIO_get_data' function. */ - /* #undef HAVE_BIO_GET_DATA */ -+#define HAVE_BIO_GET_DATA 1 - - /* Define to 1 if you have the `BIO_meth_new' function. */ - /* #undef HAVE_BIO_METH_NEW */ -+#define HAVE_BIO_METH_NEW 1 - - /* Define to 1 if you have the `cbrt' function. */ - //#define HAVE_CBRT 1 From 169ce222289b9a23079276345830cadaa0b281af Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 15 Apr 2019 02:07:03 +0100 Subject: [PATCH 05/17] Build psycopg from Python --- .appveyor.yml | 14 ++++---------- scripts/appveyor.py | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 8ad507e5..0ffa7323 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -82,23 +82,17 @@ init: # Repository gets cloned, Cache is restored + install: - "%PYEXE% C:\\appveyor.py install" +# PostgreSQL server starts now + build: off -#before_build: - build_script: - # Add PostgreSQL binaries to the path - - PATH=C:\Program Files\PostgreSQL\9.6\bin\;%PATH% - - CD C:\Project - - "%PYTHON%\\python.exe setup.py build_ext --have-ssl --pg-config %PGTOP%\\bin\\pg_config.exe -l libpgcommon -l libpgport -L %OPENSSLTOP%\\lib -I %OPENSSLTOP%\\include" - - "%PYTHON%\\python.exe setup.py build" - - "%PYTHON%\\python.exe setup.py install" - - RD /S /Q psycopg2.egg-info + - "%PYEXE% C:\\appveyor.py build_script" -#after_build: before_test: # Create and setup PostgreSQL database for the tests diff --git a/scripts/appveyor.py b/scripts/appveyor.py index 4ed758d0..896bfc9a 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -95,14 +95,11 @@ def setup_env(): def python_info(): logger.info("Python Information") - out = out_command([py_exe(), '--version'], stderr=sp.STDOUT) - logger.info("%s", out) - - out = out_command( + run_command([py_exe(), '--version'], stderr=sp.STDOUT) + run_command( [py_exe(), '-c'] + ["import sys; print('64bit: %s' % (sys.maxsize > 2**32))"] ) - logger.info("%s", out) def step_init(): @@ -131,6 +128,10 @@ def step_install(): build_libpq() +def step_build_script(): + build_psycopg() + + def build_openssl(): top = os.path.join(base_dir(), 'openssl') if os.path.exists(os.path.join(top, 'lib', 'libssl.lib')): @@ -284,6 +285,33 @@ $config->{openssl} = "%s"; shutil.rmtree(os.path.join(pgbuild)) +def build_psycopg(): + # Add PostgreSQL binaries to the path + setenv( + 'PATH', + os.pathsep.join( + [r'C:\Program Files\PostgreSQL\9.6\bin', os.environ['PATH']] + ), + ) + os.chdir(r"C:\Project") + + # Find the pg_config just built + path = os.pathsep.join( + [os.path.join(base_dir(), r'postgresql\bin'), os.environ['PATH']] + ) + setenv('PATH', path) + + run_command( + [py_exe(), "setup.py", "build_ext", "--have-ssl"] + + ["-l", "libpgcommon", "-l", "libpgport"] + + ["-L", os.path.join(base_dir(), r'openssl\lib')] + + ['-I', os.path.join(base_dir(), r'openssl\include')] + ) + run_command([py_exe(), "setup.py", "build_py"]) + run_command([py_exe(), "setup.py", "install"]) + shutil.rmtree("psycopg2.egg-info") + + def download(url, fn): """Download a file locally""" logger.info("downloading %s", url) From 5858b0b9b422470b3e54e41b0fa8cd0e604c398d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 15 Apr 2019 02:26:18 +0100 Subject: [PATCH 06/17] Test packages from Python --- .appveyor.yml | 12 ++++------ scripts/appveyor.py | 53 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 0ffa7323..809775cb 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -93,18 +93,14 @@ build: off build_script: - "%PYEXE% C:\\appveyor.py build_script" +after_build: + - "%PYEXE% C:\\appveyor.py after_build" before_test: - # Create and setup PostgreSQL database for the tests - - createdb %PSYCOPG2_TESTDB% - - psql -d %PSYCOPG2_TESTDB% -c "CREATE EXTENSION HSTORE;" + - "%PYEXE% C:\\appveyor.py before_test" test_script: - # Print psycopg and libpq versions - - "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.__version__)\"" - - "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.__libpq_version__)\"" - - "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.extensions.libpq_version())\"" - - "%PYTHON%\\python.exe -c \"import tests; tests.unittest.main(defaultTest='tests.test_suite')\" --verbose" + - "%PYEXE% C:\\appveyor.py test_script" # vim: set ts=4 sts=4 sw=4: diff --git a/scripts/appveyor.py b/scripts/appveyor.py index 896bfc9a..f10775b3 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -43,8 +43,6 @@ def setup_env(): """ Set the environment variables according to the build environment """ - python_info() - setenv('VS_VER', vs_ver()) if vs_ver() == '10.0' and opt.arch_64: @@ -103,6 +101,8 @@ def python_info(): def step_init(): + python_info() + # The program rc.exe on 64bit with some versions look in the wrong path # location when building postgresql. This cheats by copying the x64 bit # files to that location. @@ -124,11 +124,14 @@ def step_init(): def step_install(): # TODO: enable again - # build_openssl() + return + build_openssl() build_libpq() def step_build_script(): + # TODO: enable again + return build_psycopg() @@ -286,13 +289,6 @@ $config->{openssl} = "%s"; def build_psycopg(): - # Add PostgreSQL binaries to the path - setenv( - 'PATH', - os.pathsep.join( - [r'C:\Program Files\PostgreSQL\9.6\bin', os.environ['PATH']] - ), - ) os.chdir(r"C:\Project") # Find the pg_config just built @@ -312,6 +308,43 @@ def build_psycopg(): shutil.rmtree("psycopg2.egg-info") +def step_before_test(): + # Add PostgreSQL binaries to the path + setenv( + 'PATH', + os.pathsep.join([os.path.join(pg_dir(), 'bin'), os.environ['PATH']]), + ) + + # Create and setup PostgreSQL database for the tests + run_command(['createdb', os.environ['PSYCOPG2_TESTDB']]) + run_command( + ['psql', '-d', os.environ['PSYCOPG2_TESTDB']] + + ['-c', "CREATE EXTENSION hstore"] + ) + + +def step_after_build(): + # Print psycopg and libpq versions + # TODO: enable + return + + for expr in ( + 'psycopg2.__version__', + 'psycopg2.__libpq_version__', + 'psycopg2.extensions.libpq_version()', + ): + out = out_command([py_exe(), '-c', f"import psycopg2; print({expr})"]) + logger.info("built %s: %s", expr, out.decode('ascii')) + + +def step_test_script(): + run_command( + [py_exe(), '-c'] + + ["import tests; tests.unittest.main(defaultTest='tests.test_suite')"] + + ["--verbose"] + ) + + def download(url, fn): """Download a file locally""" logger.info("downloading %s", url) From fda738c90d19b77134f6b11d493a0bb841f7924f Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 15 Apr 2019 02:37:39 +0100 Subject: [PATCH 07/17] All together now, let's make this real --- .appveyor.yml | 27 --------------------------- scripts/appveyor.py | 6 ------ 2 files changed, 33 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 809775cb..0a8bcde6 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -54,33 +54,6 @@ init: - curl -fsSL -o "C:\\appveyor.py" https://raw.githubusercontent.com/psycopg/psycopg2/%APPVEYOR_REPO_COMMIT%/scripts/appveyor.py - "%PYEXE% C:\\appveyor.py init" - # TODO: exporting variables below to be dropped once all the steps are - # moved to the appveyor.py script. - - # Set env variable according to the build environment - - SET PYTHON=C:\Python%PYVER% - - IF "%PYTHON_ARCH%"=="64" SET PYTHON=%PYTHON%-x64 - - # Py 2.7 = VS Ver. 9.0 (VS 2008) - # Py 3.4 = VS Ver. 10.0 (VS 2010) - # Py 3.5, 3.6, 3.7 = VS Ver. 14.0 (VS 2015) - - IF "%PYVER%"=="27" SET VS_VER=9.0 - - IF "%PYVER%"=="34" SET VS_VER=10.0 - - IF "%PYVER%"=="35" SET VS_VER=14.0 - - IF "%PYVER%"=="36" SET VS_VER=14.0 - - IF "%PYVER%"=="37" SET VS_VER=14.0 - - - IF "%VS_VER%"=="10.0" IF "%PYTHON_ARCH%"=="64" SET DISTUTILS_USE_SDK=1 - - # Set Python to the path - - SET PATH=%PYTHON%;%PYTHON%\Scripts;C:\Program Files\Git\mingw64\bin;%PATH% - - # Setup the compiler based upon version and architecture - - ECHO Configuring Compiler - - IF "%PYTHON_ARCH%"=="32" (CALL "C:\\Program Files (x86)\\Microsoft Visual Studio %VS_VER%\\VC\\vcvarsall.bat" x86) - - IF "%PYTHON_ARCH%"=="64" (CALL "C:\\Program Files (x86)\\Microsoft Visual Studio %VS_VER%\\VC\\vcvarsall.bat" amd64) - - # Repository gets cloned, Cache is restored install: diff --git a/scripts/appveyor.py b/scripts/appveyor.py index f10775b3..dda6c583 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -123,15 +123,11 @@ def step_init(): def step_install(): - # TODO: enable again - return build_openssl() build_libpq() def step_build_script(): - # TODO: enable again - return build_psycopg() @@ -325,8 +321,6 @@ def step_before_test(): def step_after_build(): # Print psycopg and libpq versions - # TODO: enable - return for expr in ( 'psycopg2.__version__', From 591476621caf6786a84606817982bfb97a375238 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 15 Apr 2019 03:51:31 +0100 Subject: [PATCH 08/17] Dropped problematic init step It was performed before repos cloned so no resource available (including the script!) --- .appveyor.yml | 15 ++++++--------- scripts/appveyor.py | 29 +++++++++++------------------ 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 0a8bcde6..96991a69 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -49,31 +49,28 @@ cache: - C:\Others -> scripts\appveyor.cache_rebuild # Script called before repo cloning -init: - # TODO: move functionalities out of init to drop this fetch problem - - curl -fsSL -o "C:\\appveyor.py" https://raw.githubusercontent.com/psycopg/psycopg2/%APPVEYOR_REPO_COMMIT%/scripts/appveyor.py - - "%PYEXE% C:\\appveyor.py init" +# init: # Repository gets cloned, Cache is restored install: - - "%PYEXE% C:\\appveyor.py install" + - "%PYEXE% scripts\\appveyor.py install" # PostgreSQL server starts now build: off build_script: - - "%PYEXE% C:\\appveyor.py build_script" + - "%PYEXE% scripts\\appveyor.py build_script" after_build: - - "%PYEXE% C:\\appveyor.py after_build" + - "%PYEXE% scripts\\appveyor.py after_build" before_test: - - "%PYEXE% C:\\appveyor.py before_test" + - "%PYEXE% scripts\\appveyor.py before_test" test_script: - - "%PYEXE% C:\\appveyor.py test_script" + - "%PYEXE% scripts\\appveyor.py test_script" # vim: set ts=4 sts=4 sw=4: diff --git a/scripts/appveyor.py b/scripts/appveyor.py index dda6c583..2fdb3606 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -64,23 +64,15 @@ def setup_env(): ) # Fix problem with VS2010 Express 64bit missing vcvars64.bat - # Note: repository not cloned at this point, so need to fetch - # file another way if vs_ver() == '10.0': if not os.path.exists( os.path.join(vc_dir(), r"bin\amd64\vcvars64.bat") ): logger.info("Fixing VS2010 Express and 64bit builds") - with urlopen( - "https://raw.githubusercontent.com/psycopg/psycopg2/" - "master/scripts/vcvars64-vs2010.bat" - ) as f: - data = f.read() - - with open( - os.path.join(vc_dir(), r"bin\amd64\vcvars64.bat"), 'w' - ) as f: - f.write(data) + shutil.copy( + os.path.join(clone_dir(), r"scripts\vcvars64-vs2010.bat"), + os.path.join(vc_dir(), r"bin\amd64\vcvars64.bat"), + ) logger.info("Configuring compiler") bat_call( @@ -100,7 +92,7 @@ def python_info(): ) -def step_init(): +def step_install(): python_info() # The program rc.exe on 64bit with some versions look in the wrong path @@ -122,12 +114,9 @@ def step_init(): print("max_prepared_transactions = 10", file=f) -def step_install(): +def step_build_script(): build_openssl() build_libpq() - - -def step_build_script(): build_psycopg() @@ -285,7 +274,7 @@ $config->{openssl} = "%s"; def build_psycopg(): - os.chdir(r"C:\Project") + os.chdir(clone_dir()) # Find the pg_config just built path = os.pathsep.join( @@ -472,6 +461,10 @@ def vs_ver(pyver=None): return vsver +def clone_dir(): + return r"C:\Project" + + def pg_dir(): return r"C:\Program Files\PostgreSQL\9.6" From 5c72203180a5cbe0970a463f7fe6f1138db12bba Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 15 Apr 2019 03:58:38 +0100 Subject: [PATCH 09/17] Using pathlib to manipulate paths --- scripts/appveyor.py | 138 +++++++++++++++++++++++--------------------- 1 file changed, 72 insertions(+), 66 deletions(-) diff --git a/scripts/appveyor.py b/scripts/appveyor.py index 2fdb3606..e84e9af4 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -15,6 +15,7 @@ import shutil import logging import subprocess as sp from glob import glob +from pathlib import Path from zipfile import ZipFile from tempfile import NamedTemporaryFile from functools import lru_cache @@ -49,8 +50,8 @@ def setup_env(): setenv('DISTUTILS_USE_SDK', '1') path = [ - py_dir(), - os.path.join(py_dir(), 'Scripts'), + str(py_dir()), + str(py_dir() / 'Scripts'), r'C:\Program Files\Git\mingw64\bin', os.environ['PATH'], ] @@ -59,28 +60,21 @@ def setup_env(): if vs_ver() == '9.0': logger.info("Fixing VS2008 Express and 64bit builds") shutil.copyfile( - os.path.join(vc_dir(), r"bin\vcvars64.bat"), - os.path.join(vc_dir(), r"bin\amd64\vcvarsamd64.bat"), + vc_dir() / r"bin\vcvars64.bat", + vc_dir() / r"bin\amd64\vcvarsamd64.bat", ) # Fix problem with VS2010 Express 64bit missing vcvars64.bat if vs_ver() == '10.0': - if not os.path.exists( - os.path.join(vc_dir(), r"bin\amd64\vcvars64.bat") - ): + if not (vc_dir() / r"bin\amd64\vcvars64.bat").exists(): logger.info("Fixing VS2010 Express and 64bit builds") - shutil.copy( - os.path.join(clone_dir(), r"scripts\vcvars64-vs2010.bat"), - os.path.join(vc_dir(), r"bin\amd64\vcvars64.bat"), + copy_file( + clone_dir() / r"scripts\vcvars64-vs2010.bat", + vc_dir() / r"bin\amd64\vcvars64.bat", ) logger.info("Configuring compiler") - bat_call( - [ - os.path.join(vc_dir(), "vcvarsall.bat"), - 'x86' if opt.arch_32 else 'amd64', - ] - ) + bat_call([vc_dir() / "vcvarsall.bat", 'x86' if opt.arch_32 else 'amd64']) def python_info(): @@ -102,14 +96,13 @@ def step_install(): for fn in glob( r'C:\Program Files\Microsoft SDKs\Windows\v7.0\Bin\x64\rc*' ): - logger.info("Copying %s to a better place" % os.path.basename(fn)) - shutil.copy( + copy_file( fn, r"C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Bin" ) # Change PostgreSQL config before service starts logger.info("Configuring Postgres") - with open(os.path.join(pg_dir(), 'data', 'postgresql.conf'), 'a') as f: + with (pg_dir() / 'data' / 'postgresql.conf').open('a') as f: # allow > 1 prepared transactions for test cases print("max_prepared_transactions = 10", file=f) @@ -121,15 +114,15 @@ def step_build_script(): def build_openssl(): - top = os.path.join(base_dir(), 'openssl') - if os.path.exists(os.path.join(top, 'lib', 'libssl.lib')): + top = base_dir() / 'openssl' + if (top / 'lib' / 'libssl.lib').exists(): return logger.info("Building OpenSSL") # Setup directories for building OpenSSL libraries - ensure_dir(os.path.join(top, 'include', 'openssl')) - ensure_dir(os.path.join(top, 'lib')) + ensure_dir(top / 'include' / 'openssl') + ensure_dir(top / 'lib') # Setup OpenSSL Environment Variables based on processor architecture if opt.arch_32: @@ -144,8 +137,8 @@ def build_openssl(): # Download OpenSSL source zipname = f'OpenSSL_{ver}.zip' - zipfile = os.path.join(r'C:\Others', zipname) - if not os.path.exists(zipfile): + zipfile = top_dir() / zipname + if not zipfile.exists(): download( f"https://github.com/openssl/openssl/archive/{zipname}", zipfile ) @@ -153,7 +146,7 @@ def build_openssl(): with ZipFile(zipfile) as z: z.extractall(path=build_dir()) - os.chdir(os.path.join(build_dir(), f"openssl-OpenSSL_{ver}")) + os.chdir(build_dir() / f"openssl-OpenSSL_{ver}") run_command( ['perl', 'Configure', target, 'no-asm'] + ['no-shared', 'no-zlib', f'--prefix={top}', f'--openssldir={top}'] @@ -161,30 +154,30 @@ def build_openssl(): run_command("nmake build_libs install_dev".split()) - assert os.path.exists(os.path.join(top, 'lib', 'libssl.lib')) + assert (top / 'lib' / 'libssl.lib').exists() os.chdir(base_dir()) - shutil.rmtree(os.path.join(build_dir(), f"openssl-OpenSSL_{ver}")) + shutil.rmtree(build_dir() / f"openssl-OpenSSL_{ver}") def build_libpq(): - top = os.path.join(base_dir(), 'postgresql') - if os.path.exists(os.path.join(top, 'lib', 'libpq.lib')): + top = base_dir() / 'postgresql' + if (top / 'lib' / 'libpq.lib').exists(): return logger.info("Building libpq") # Setup directories for building PostgreSQL librarires - ensure_dir(os.path.join(top, 'include')) - ensure_dir(os.path.join(top, 'lib')) - ensure_dir(os.path.join(top, 'bin')) + ensure_dir(top / 'include') + ensure_dir(top / 'lib') + ensure_dir(top / 'bin') ver = os.environ['POSTGRES_VERSION'] # Download PostgreSQL source zipname = f'postgres-REL_{ver}.zip' - zipfile = os.path.join(r'C:\Others', zipname) - if not os.path.exists(zipfile): + zipfile = top_dir() / zipname + if not zipfile.exists(): download( f"https://github.com/postgres/postgres/archive/REL_{ver}.zip", zipfile, @@ -193,12 +186,12 @@ def build_libpq(): with ZipFile(zipfile) as z: z.extractall(path=build_dir()) - pgbuild = os.path.join(build_dir(), f"postgres-REL_{ver}") + pgbuild = build_dir() / f"postgres-REL_{ver}" os.chdir(pgbuild) # Patch for OpenSSL 1.1 configuration. See: # https://www.postgresql-archive.org/Compile-psql-9-6-with-SSL-Version-1-1-0-td6054118.html - assert os.path.exists("src/include/pg_config.h.win32") + assert Path("src/include/pg_config.h.win32").exists() with open("src/include/pg_config.h.win32", 'a') as f: print( """ @@ -220,7 +213,7 @@ $config->{openssl} = "%s"; 1; """ - % os.path.join(base_dir(), 'openssl').replace('\\', '\\\\'), + % str(base_dir() / 'openssl').replace('\\', '\\\\'), file=f, ) @@ -233,7 +226,7 @@ $config->{openssl} = "%s"; run_command([which("build"), "libpq"]) # Install includes - with open(os.path.join(pgbuild, "src/backend/parser/gram.h"), "w") as f: + with (pgbuild / "src/backend/parser/gram.h").open("w") as f: print("", file=f) # Copy over built libraries @@ -244,20 +237,14 @@ $config->{openssl} = "%s"; ) for lib in ('libpgport', 'libpgcommon', 'libpq'): - shutil.copy( - os.path.join(pgbuild, f'Release/{lib}/{lib}.lib'), - os.path.join(top, 'lib'), - ) + copy_file(pgbuild / f'Release/{lib}/{lib}.lib', top / 'lib') # Prepare local include directory for building from for dir in ('win32', 'win32_msvc'): - merge_dir( - os.path.join(pgbuild, f"src/include/port/{dir}"), - os.path.join(pgbuild, "src/include"), - ) + merge_dir(pgbuild / f"src/include/port/{dir}", pgbuild / "src/include") # Build pg_config in place - os.chdir(os.path.join(pgbuild, 'src/bin/pg_config')) + os.chdir(pgbuild / 'src/bin/pg_config') run_command( ['cl', 'pg_config.c', '/MT', '/nologo', fr'/I{pgbuild}\src\include'] + ['/link', fr'/LIBPATH:{top}\lib'] @@ -266,11 +253,11 @@ $config->{openssl} = "%s"; + [fr'/OUT:{top}\bin\pg_config.exe'] ) - assert os.path.exists(os.path.join(top, 'lib', 'libpq.lib')) - assert os.path.exists(os.path.join(top, 'bin', 'pg_config.exe')) + assert (top / 'lib' / 'libpq.lib').exists() + assert (top / 'bin' / 'pg_config.exe').exists() os.chdir(base_dir()) - shutil.rmtree(os.path.join(pgbuild)) + shutil.rmtree(pgbuild) def build_psycopg(): @@ -278,15 +265,15 @@ def build_psycopg(): # Find the pg_config just built path = os.pathsep.join( - [os.path.join(base_dir(), r'postgresql\bin'), os.environ['PATH']] + [str(base_dir() / r'postgresql\bin'), os.environ['PATH']] ) setenv('PATH', path) run_command( [py_exe(), "setup.py", "build_ext", "--have-ssl"] + ["-l", "libpgcommon", "-l", "libpgport"] - + ["-L", os.path.join(base_dir(), r'openssl\lib')] - + ['-I', os.path.join(base_dir(), r'openssl\include')] + + ["-L", base_dir() / r'openssl\lib'] + + ['-I', base_dir() / r'openssl\include'] ) run_command([py_exe(), "setup.py", "build_py"]) run_command([py_exe(), "setup.py", "install"]) @@ -296,8 +283,7 @@ def build_psycopg(): def step_before_test(): # Add PostgreSQL binaries to the path setenv( - 'PATH', - os.pathsep.join([os.path.join(pg_dir(), 'bin'), os.environ['PATH']]), + 'PATH', os.pathsep.join([str(pg_dir() / 'bin'), os.environ['PATH']]) ) # Create and setup PostgreSQL database for the tests @@ -359,6 +345,7 @@ def merge_dir(src, tgt): Reproduce the semantic of "XCOPY /Y /S src/* tgt" """ + src = str(src) for dp, _dns, fns in os.walk(src): logger.debug("dirpath %s", dp) if not fns: @@ -367,7 +354,7 @@ def merge_dir(src, tgt): subdir = dp[len(src) :].lstrip(os.sep) tgtdir = ensure_dir(os.path.join(tgt, subdir)) for fn in fns: - shutil.copy(os.path.join(dp, fn), tgtdir) + copy_file(os.path.join(dp, fn), tgtdir) def bat_call(cmdline): @@ -381,6 +368,7 @@ def bat_call(cmdline): """ if not isinstance(cmdline, str): + cmdline = map(str, cmdline) cmdline = ' '.join(c if ' ' not in c else '"%s"' % c for c in cmdline) pyexe = py_exe() @@ -422,14 +410,14 @@ def py_dir(): Return the path to the target python binary to execute. """ dirname = ''.join([r"C:\Python", opt.pyver, '-x64' if opt.arch_64 else '']) - return dirname + return Path(dirname) def py_exe(): """ Return the full path of the target python executable. """ - return os.path.join(py_dir(), 'python.exe') + return py_dir() / 'python.exe' def vc_dir(vsver=None): @@ -439,7 +427,9 @@ def vc_dir(vsver=None): if vsver is None: vsver = vs_ver() - return r"C:\Program Files (x86)\Microsoft Visual Studio %s\VC" % vsver + return Path( + r"C:\Program Files (x86)\Microsoft Visual Studio %s\VC" % vsver + ) def vs_ver(pyver=None): @@ -462,42 +452,58 @@ def vs_ver(pyver=None): def clone_dir(): - return r"C:\Project" + return Path(r"C:\Project") def pg_dir(): - return r"C:\Program Files\PostgreSQL\9.6" + return Path(r"C:\Program Files\PostgreSQL\9.6") + + +def top_dir(): + return Path(r"C:\Others") def base_dir(): - rv = r"C:\Others\%s\%s" % (opt.pyarch, vs_ver()) + rv = top_dir() / opt.pyarch / vs_ver() return ensure_dir(rv) def build_dir(): - rv = os.path.join(base_dir(), 'Builds') + rv = base_dir() / 'Builds' return ensure_dir(rv) def ensure_dir(dir): - if not os.path.exists(dir): + if not isinstance(dir, Path): + dir = Path(dir) + + if not dir.is_dir(): logger.info("creating directory %s", dir) - os.makedirs(dir) + dir.mkdir(parents=True) return dir def run_command(cmdline, **kwargs): + if not isinstance(cmdline, str): + cmdline = list(map(str, cmdline)) logger.debug("calling command: %s", cmdline) sp.check_call(cmdline, **kwargs) def out_command(cmdline, **kwargs): + if not isinstance(cmdline, str): + cmdline = list(map(str, cmdline)) logger.debug("calling command: %s", cmdline) data = sp.check_output(cmdline, **kwargs) return data +def copy_file(src, dst): + logger.info("copying file %s -> %s", src, dst) + shutil.copy(src, dst) + + def setenv(k, v): logger.debug("setting %s=%s", k, v) os.environ[k] = v From 1178501aaf969ece80a98887cf399541af13de21 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 21 Apr 2019 12:41:09 +0100 Subject: [PATCH 10/17] appveyor: added logging level configuration --- scripts/appveyor.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/scripts/appveyor.py b/scripts/appveyor.py index e84e9af4..72a20631 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -26,13 +26,14 @@ STEP_PREFIX = 'step_' logger = logging.getLogger() logging.basicConfig( - level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s' + level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s' ) def main(): global opt opt = parse_cmdline() + logger.setLevel(opt.loglevel) setup_env() cmd = globals()[STEP_PREFIX + opt.step] @@ -487,14 +488,14 @@ def ensure_dir(dir): def run_command(cmdline, **kwargs): if not isinstance(cmdline, str): cmdline = list(map(str, cmdline)) - logger.debug("calling command: %s", cmdline) + logger.info("running command: %s", cmdline) sp.check_call(cmdline, **kwargs) def out_command(cmdline, **kwargs): if not isinstance(cmdline, str): cmdline = list(map(str, cmdline)) - logger.debug("calling command: %s", cmdline) + logger.info("running command: %s", cmdline) data = sp.check_output(cmdline, **kwargs) return data @@ -545,6 +546,26 @@ def parse_cmdline(): help="the target python architecture. Default from PYTHON_ARCH env var", ) + g = parser.add_mutually_exclusive_group() + g.add_argument( + '-q', + '--quiet', + help="Talk less", + dest='loglevel', + action='store_const', + const=logging.WARN, + default=logging.INFO, + ) + g.add_argument( + '-v', + '--verbose', + help="Talk more", + dest='loglevel', + action='store_const', + const=logging.DEBUG, + default=logging.INFO, + ) + steps = [ n[len(STEP_PREFIX) :] for n in globals() From b1078b1b92534ea1ba9e72eb84b36531893b9ba3 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 21 Apr 2019 13:14:37 +0100 Subject: [PATCH 11/17] Setup build environment only before building --- scripts/appveyor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/appveyor.py b/scripts/appveyor.py index 72a20631..0992b7d7 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -35,7 +35,6 @@ def main(): opt = parse_cmdline() logger.setLevel(opt.loglevel) - setup_env() cmd = globals()[STEP_PREFIX + opt.step] cmd() @@ -109,6 +108,7 @@ def step_install(): def step_build_script(): + setup_env() build_openssl() build_libpq() build_psycopg() From 9eec303cf7989f7e8865ce87ff36c7f676a88fff Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 21 Apr 2019 13:33:15 +0100 Subject: [PATCH 12/17] Configure postgres to run appveyor tests on ssl --- .appveyor.yml | 7 ++++-- scripts/appveyor.py | 59 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 96991a69..c8b03c5d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -31,6 +31,10 @@ environment: PGUSER: postgres PGPASSWORD: Password12! + PGSSLMODE: require + + # Select according to the service enabled + POSTGRES_DIR: C:\Program Files\PostgreSQL\9.6\ # The python used in the build process, not the one packages are built for PYEXE: C:\Python36\python.exe @@ -39,8 +43,7 @@ matrix: fast_finish: false services: - # Note: if you change this service also change the paths to match - # (see where Program Files\Postgres\9.6 is used) + # Note: if you change this service also change POSTGRES_DIR - postgresql96 cache: diff --git a/scripts/appveyor.py b/scripts/appveyor.py index 0992b7d7..9417d67e 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -88,7 +88,11 @@ def python_info(): def step_install(): python_info() + configure_sdk() + configure_postgres() + +def configure_sdk(): # The program rc.exe on 64bit with some versions look in the wrong path # location when building postgresql. This cheats by copying the x64 bit # files to that location. @@ -100,11 +104,50 @@ def step_install(): fn, r"C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Bin" ) + +def configure_postgres(): # Change PostgreSQL config before service starts logger.info("Configuring Postgres") - with (pg_dir() / 'data' / 'postgresql.conf').open('a') as f: + with (pg_data_dir() / 'postgresql.conf').open('a') as f: # allow > 1 prepared transactions for test cases print("max_prepared_transactions = 10", file=f) + print("ssl = on", file=f) + + # Create openssl certificate to allow ssl connection + cwd = os.getcwd() + os.chdir(pg_data_dir()) + run_openssl( + 'req -new -x509 -days 365 -nodes -text ' + '-out server.crt -keyout server.key -subj /CN=initd.org'.split() + ) + run_openssl( + 'req -new -nodes -text -out root.csr -keyout root.key ' + '-subj /CN=initd.org'.split() + ) + + run_openssl( + 'x509 -req -in root.csr -text -days 3650 -extensions v3_ca ' + '-signkey root.key -out root.crt'.split() + ) + + run_openssl( + 'req -new -nodes -text -out server.csr -keyout server.key ' + '-subj /CN=initd.org'.split() + ) + + run_openssl( + 'x509 -req -in server.csr -text -days 365 -CA root.crt ' + '-CAkey root.key -CAcreateserial -out server.crt'.split() + ) + + os.chdir(cwd) + + +def run_openssl(args): + """Run the appveyor-installed openssl""" + # https://www.appveyor.com/docs/windows-images-software/ + openssl = Path(r"C:\OpenSSL-v111-Win64") / 'bin' / 'openssl' + return run_command([openssl] + args) def step_build_script(): @@ -283,9 +326,7 @@ def build_psycopg(): def step_before_test(): # Add PostgreSQL binaries to the path - setenv( - 'PATH', os.pathsep.join([str(pg_dir() / 'bin'), os.environ['PATH']]) - ) + setenv('PATH', os.pathsep.join([str(pg_bin_dir()), os.environ['PATH']])) # Create and setup PostgreSQL database for the tests run_command(['createdb', os.environ['PSYCOPG2_TESTDB']]) @@ -457,7 +498,15 @@ def clone_dir(): def pg_dir(): - return Path(r"C:\Program Files\PostgreSQL\9.6") + return Path(os.environ['POSTGRES_DIR']) + + +def pg_data_dir(): + return pg_dir() / 'data' + + +def pg_bin_dir(): + return pg_dir() / 'bin' def top_dir(): From 637a990e093edf55018458120e361fc8dc0b87bf Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 21 Apr 2019 17:08:35 +0100 Subject: [PATCH 13/17] Added support for wheel building and uploading To be used by the psycopg/psycopg2-wheels project. --- scripts/appveyor.py | 247 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 225 insertions(+), 22 deletions(-) diff --git a/scripts/appveyor.py b/scripts/appveyor.py index 9417d67e..4a0e8154 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -18,7 +18,6 @@ from glob import glob from pathlib import Path from zipfile import ZipFile from tempfile import NamedTemporaryFile -from functools import lru_cache from urllib.request import urlopen opt = None @@ -39,8 +38,7 @@ def main(): cmd() -@lru_cache() -def setup_env(): +def setup_build_env(): """ Set the environment variables according to the build environment """ @@ -69,7 +67,7 @@ def setup_env(): if not (vc_dir() / r"bin\amd64\vcvars64.bat").exists(): logger.info("Fixing VS2010 Express and 64bit builds") copy_file( - clone_dir() / r"scripts\vcvars64-vs2010.bat", + package_dir() / r"scripts\vcvars64-vs2010.bat", vc_dir() / r"bin\amd64\vcvars64.bat", ) @@ -91,6 +89,17 @@ def step_install(): configure_sdk() configure_postgres() + if is_wheel(): + install_wheel_support() + + +def install_wheel_support(): + """ + Install an up-to-date pip wheel package to build wheels. + """ + run_command([py_exe()] + "-m pip install --upgrade pip".split()) + run_command([py_exe()] + "-m pip install wheel".split()) + def configure_sdk(): # The program rc.exe on 64bit with some versions look in the wrong path @@ -106,7 +115,9 @@ def configure_sdk(): def configure_postgres(): - # Change PostgreSQL config before service starts + """ + Set up PostgreSQL config before the service starts. + """ logger.info("Configuring Postgres") with (pg_data_dir() / 'postgresql.conf').open('a') as f: # allow > 1 prepared transactions for test cases @@ -144,18 +155,21 @@ def configure_postgres(): def run_openssl(args): - """Run the appveyor-installed openssl""" + """Run the appveyor-installed openssl with some args.""" # https://www.appveyor.com/docs/windows-images-software/ openssl = Path(r"C:\OpenSSL-v111-Win64") / 'bin' / 'openssl' return run_command([openssl] + args) def step_build_script(): - setup_env() + setup_build_env() build_openssl() build_libpq() build_psycopg() + if is_wheel(): + build_binary_packages() + def build_openssl(): top = base_dir() / 'openssl' @@ -305,14 +319,9 @@ $config->{openssl} = "%s"; def build_psycopg(): - os.chdir(clone_dir()) - - # Find the pg_config just built - path = os.pathsep.join( - [str(base_dir() / r'postgresql\bin'), os.environ['PATH']] - ) - setenv('PATH', path) - + os.chdir(package_dir()) + patch_package_name() + add_pg_config_path() run_command( [py_exe(), "setup.py", "build_ext", "--have-ssl"] + ["-l", "libpgcommon", "-l", "libpgport"] @@ -320,25 +329,87 @@ def build_psycopg(): + ['-I', base_dir() / r'openssl\include'] ) run_command([py_exe(), "setup.py", "build_py"]) + + +def patch_package_name(): + """Change the psycopg2 package name in the setup.py if required.""" + conf = os.environ.get('CONFIGURATION', 'psycopg2') + if conf == 'psycopg2': + return + + logger.info("changing package name to %s", conf) + + with (package_dir() / 'setup.py').open() as f: + data = f.read() + + # Replace the name of the package with what desired + rex = re.compile(r"""name=["']psycopg2["']""") + assert len(rex.findall(data)) == 1, rex.findall(data) + data = rex.sub(f'name="{conf}"', data) + + with (package_dir() / 'setup.py').open('w') as f: + f.write(data) + + +def build_binary_packages(): + """Create wheel/exe binary packages.""" + os.chdir(package_dir()) + + add_pg_config_path() + + # Build .exe packages for whom still use them + if os.environ['CONFIGURATION'] == 'psycopg2': + run_command([py_exe(), 'setup.py', 'bdist_wininst', "-d", dist_dir()]) + + # Build .whl packages + run_command([py_exe(), 'setup.py', 'bdist_wheel', "-d", dist_dir()]) + + +def step_after_build(): + if not is_wheel(): + install_built_package() + else: + install_binary_package() + + +def install_built_package(): + """Install the package just built by setup build.""" + os.chdir(package_dir()) + + # Install the psycopg just built + add_pg_config_path() run_command([py_exe(), "setup.py", "install"]) shutil.rmtree("psycopg2.egg-info") +def install_binary_package(): + """Install the package from a packaged wheel.""" + run_command( + [py_exe(), '-m', 'pip', 'install', '--no-index', '-f', dist_dir()] + + [os.environ['CONFIGURATION']] + ) + + +def add_pg_config_path(): + """Allow finding in the path the pg_config just built.""" + pg_path = str(base_dir() / r'postgresql\bin') + if pg_path not in os.environ['PATH'].split(os.pathsep): + setenv('PATH', os.pathsep.join([pg_path, os.environ['PATH']])) + + def step_before_test(): - # Add PostgreSQL binaries to the path - setenv('PATH', os.pathsep.join([str(pg_bin_dir()), os.environ['PATH']])) + print_psycopg2_version() # Create and setup PostgreSQL database for the tests - run_command(['createdb', os.environ['PSYCOPG2_TESTDB']]) + run_command([pg_bin_dir() / 'createdb', os.environ['PSYCOPG2_TESTDB']]) run_command( - ['psql', '-d', os.environ['PSYCOPG2_TESTDB']] + [pg_bin_dir() / 'psql', '-d', os.environ['PSYCOPG2_TESTDB']] + ['-c', "CREATE EXTENSION hstore"] ) -def step_after_build(): - # Print psycopg and libpq versions - +def print_psycopg2_version(): + """Print psycopg2 and libpq versions installed.""" for expr in ( 'psycopg2.__version__', 'psycopg2.__libpq_version__', @@ -349,6 +420,36 @@ def step_after_build(): def step_test_script(): + check_libpq_version() + run_test_suite() + + +def check_libpq_version(): + """ + Fail if the package installed is not using the expected libpq version. + """ + want_ver = tuple(map(int, os.environ['POSTGRES_VERSION'].split('_'))) + want_ver = "%d%04d" % want_ver + got_ver = ( + out_command( + [py_exe(), '-c'] + + ["import psycopg2; print(psycopg2.extensions.libpq_version())"] + ) + .decode('ascii') + .rstrip() + ) + assert want_ver == got_ver, "libpq version mismatch: %r != %r" % ( + want_ver, + got_ver, + ) + + +def run_test_suite(): + # Remove this var, which would make badly a configured OpenSSL 1.1 work + os.environ.pop('OPENSSL_CONF', None) + + # Run the unit test + os.chdir(package_dir()) run_command( [py_exe(), '-c'] + ["import tests; tests.unittest.main(defaultTest='tests.test_suite')"] @@ -356,6 +457,72 @@ def step_test_script(): ) +def step_on_success(): + print_sha1_hashes() + if setup_ssh(): + upload_packages() + + +def print_sha1_hashes(): + """ + Print the packages sha1 so their integrity can be checked upon signing. + """ + logger.info("artifacts SHA1 hashes:") + + os.chdir(package_dir() / 'dist') + run_command([which('sha1sum'), '-b', f'psycopg2-*/*']) + + +def setup_ssh(): + """ + Configure ssh to upload built packages where they can be retrieved. + + Return False if can't configure and upload shoould be skipped. + """ + # If we are not on the psycopg AppVeyor account, the environment variable + # REMOTE_KEY will not be decrypted. In that case skip uploading. + if os.environ['APPVEYOR_ACCOUNT_NAME'] != 'psycopg': + logger.warn("skipping artifact upload: you are not psycopg") + return False + + pkey = os.environ.get('REMOTE_KEY', None) + if not pkey: + logger.warn("skipping artifact upload: no remote key") + return False + + # Write SSH Private Key file from environment variable + pkey = pkey.replace(' ', '\n') + with (clone_dir() / 'id_rsa').open('w') as f: + f.write( + f"""\ +-----BEGIN RSA PRIVATE KEY----- +{pkey} +-----END RSA PRIVATE KEY----- +""" + ) + + # Make a directory to please MinGW's version of ssh + ensure_dir(r"C:\MinGW\msys\1.0\home\appveyor\.ssh") + + return True + + +def upload_packages(): + # Upload built artifacts + logger.info("uploading artifacts") + + ssh_cmd = r"C:\MinGW\msys\1.0\bin\ssh -i %s -o UserKnownHostsFile=%s" % ( + clone_dir() / "id_rsa", + clone_dir() / 'known_hosts', + ) + + os.chdir(package_dir()) + run_command( + [r"C:\MinGW\msys\1.0\bin\rsync", "-avr"] + + ["-e", ssh_cmd, "dist/", "upload@initd.org:"] + ) + + def download(url, fn): """Download a file locally""" logger.info("downloading %s", url) @@ -523,6 +690,42 @@ def build_dir(): return ensure_dir(rv) +def package_dir(): + """ + Return the directory containing the psycopg code checkout dir + + Building psycopg is clone_dir(), building the wheel packages is a submodule. + """ + return clone_dir() / 'psycopg2' if is_wheel() else clone_dir() + + +def is_wheel(): + """ + Return whether we are building the wheel packages or just the extension. + """ + project_name = os.environ['APPVEYOR_PROJECT_NAME'] + if project_name == 'psycopg2': + return False + elif project_name == 'psycopg2-wheels': + return True + else: + raise Exception(f"unexpected project name: {project_name}") + + +def dist_dir(): + return package_dir() / 'dist' / ('psycopg2-%s' % package_version()) + + +def package_version(): + with (package_dir() / 'setup.py').open() as f: + data = f.read() + + m = re.search( + r"""^PSYCOPG_VERSION\s*=\s*['"](.*)['"]""", data, re.MULTILINE + ) + return m.group(1) + + def ensure_dir(dir): if not isinstance(dir, Path): dir = Path(dir) From 285c64d101b9b392d5d7d88639267bc9b73224a1 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 22 Apr 2019 03:10:31 +0100 Subject: [PATCH 14/17] Better dir names in appveyor build --- scripts/appveyor.py | 49 +++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/scripts/appveyor.py b/scripts/appveyor.py index 4a0e8154..737ff480 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -172,7 +172,7 @@ def step_build_script(): def build_openssl(): - top = base_dir() / 'openssl' + top = ssl_build_dir() if (top / 'lib' / 'libssl.lib').exists(): return @@ -195,7 +195,7 @@ def build_openssl(): # Download OpenSSL source zipname = f'OpenSSL_{ver}.zip' - zipfile = top_dir() / zipname + zipfile = cache_dir() / zipname if not zipfile.exists(): download( f"https://github.com/openssl/openssl/archive/{zipname}", zipfile @@ -204,7 +204,8 @@ def build_openssl(): with ZipFile(zipfile) as z: z.extractall(path=build_dir()) - os.chdir(build_dir() / f"openssl-OpenSSL_{ver}") + sslbuild = build_dir() / f"openssl-OpenSSL_{ver}" + os.chdir(sslbuild) run_command( ['perl', 'Configure', target, 'no-asm'] + ['no-shared', 'no-zlib', f'--prefix={top}', f'--openssldir={top}'] @@ -214,12 +215,12 @@ def build_openssl(): assert (top / 'lib' / 'libssl.lib').exists() - os.chdir(base_dir()) - shutil.rmtree(build_dir() / f"openssl-OpenSSL_{ver}") + os.chdir(clone_dir()) + shutil.rmtree(sslbuild) def build_libpq(): - top = base_dir() / 'postgresql' + top = pg_build_dir() if (top / 'lib' / 'libpq.lib').exists(): return @@ -234,7 +235,7 @@ def build_libpq(): # Download PostgreSQL source zipname = f'postgres-REL_{ver}.zip' - zipfile = top_dir() / zipname + zipfile = cache_dir() / zipname if not zipfile.exists(): download( f"https://github.com/postgres/postgres/archive/REL_{ver}.zip", @@ -271,7 +272,7 @@ $config->{openssl} = "%s"; 1; """ - % str(base_dir() / 'openssl').replace('\\', '\\\\'), + % str(ssl_build_dir()).replace('\\', '\\\\'), file=f, ) @@ -314,7 +315,7 @@ $config->{openssl} = "%s"; assert (top / 'lib' / 'libpq.lib').exists() assert (top / 'bin' / 'pg_config.exe').exists() - os.chdir(base_dir()) + os.chdir(clone_dir()) shutil.rmtree(pgbuild) @@ -325,8 +326,8 @@ def build_psycopg(): run_command( [py_exe(), "setup.py", "build_ext", "--have-ssl"] + ["-l", "libpgcommon", "-l", "libpgport"] - + ["-L", base_dir() / r'openssl\lib'] - + ['-I', base_dir() / r'openssl\include'] + + ["-L", ssl_build_dir() / 'lib'] + + ['-I', ssl_build_dir() / 'include'] ) run_command([py_exe(), "setup.py", "build_py"]) @@ -392,7 +393,7 @@ def install_binary_package(): def add_pg_config_path(): """Allow finding in the path the pg_config just built.""" - pg_path = str(base_dir() / r'postgresql\bin') + pg_path = str(pg_build_dir() / 'bin') if pg_path not in os.environ['PATH'].split(os.pathsep): setenv('PATH', os.pathsep.join([pg_path, os.environ['PATH']])) @@ -664,29 +665,37 @@ def clone_dir(): return Path(r"C:\Project") -def pg_dir(): +def appveyor_pg_dir(): return Path(os.environ['POSTGRES_DIR']) def pg_data_dir(): - return pg_dir() / 'data' + return appveyor_pg_dir() / 'data' def pg_bin_dir(): - return pg_dir() / 'bin' + return appveyor_pg_dir() / 'bin' -def top_dir(): - return Path(r"C:\Others") +def pg_build_dir(): + return cache_arch_dir() / 'postgresql' -def base_dir(): - rv = top_dir() / opt.pyarch / vs_ver() +def ssl_build_dir(): + return cache_arch_dir() / 'openssl' + + +def cache_arch_dir(): + rv = cache_dir() / opt.pyarch / vs_ver() return ensure_dir(rv) +def cache_dir(): + return Path(r"C:\Others") + + def build_dir(): - rv = base_dir() / 'Builds' + rv = cache_arch_dir() / 'Builds' return ensure_dir(rv) From 1b2c1d620f769a87442446ece5b1915cb537e73e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 22 Apr 2019 11:10:47 +0100 Subject: [PATCH 15/17] Run tests more quiet/faster building wheels We are mostly interested it installed alright. --- scripts/appveyor.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scripts/appveyor.py b/scripts/appveyor.py index 737ff480..abd68ed2 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -450,12 +450,19 @@ def run_test_suite(): os.environ.pop('OPENSSL_CONF', None) # Run the unit test + cmdline = [ + py_exe(), + '-c', + "import tests; tests.unittest.main(defaultTest='tests.test_suite')", + ] + + if is_wheel(): + os.environ['PSYCOPG2_TEST_FAST'] = '1' + else: + cmdline.append('--verbose') + os.chdir(package_dir()) - run_command( - [py_exe(), '-c'] - + ["import tests; tests.unittest.main(defaultTest='tests.test_suite')"] - + ["--verbose"] - ) + run_command(cmdline) def step_on_success(): From 014097c1afcdbcad72fe8392a635b2a4a7b36d14 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 22 Apr 2019 12:33:03 +0100 Subject: [PATCH 16/17] Dropped command line config from appveyor scrips Only use env vars, they were unused. Use consistently a config object with properties instead of functions (the one returning a binary are especially dangerous if parens are forgotten). Also add helpers to call the target python more succinctly. --- .appveyor.yml | 20 +- scripts/appveyor.py | 449 +++++++++++++++++++++++--------------------- 2 files changed, 247 insertions(+), 222 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index c8b03c5d..13d4e251 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,16 +11,16 @@ environment: matrix: # For Python versions available on Appveyor, see # https://www.appveyor.com/docs/build-environment/ - - {PYVER: "27", PYTHON_ARCH: "32"} - - {PYVER: "27", PYTHON_ARCH: "64"} - - {PYVER: "37", PYTHON_ARCH: "32"} - - {PYVER: "37", PYTHON_ARCH: "64"} - - {PYVER: "36", PYTHON_ARCH: "32"} - - {PYVER: "36", PYTHON_ARCH: "64"} - - {PYVER: "35", PYTHON_ARCH: "32"} - - {PYVER: "35", PYTHON_ARCH: "64"} - - {PYVER: "34", PYTHON_ARCH: "32"} - - {PYVER: "34", PYTHON_ARCH: "64"} + - {PY_VER: "27", PY_ARCH: "32"} + - {PY_VER: "27", PY_ARCH: "64"} + - {PY_VER: "37", PY_ARCH: "32"} + - {PY_VER: "37", PY_ARCH: "64"} + - {PY_VER: "36", PY_ARCH: "32"} + - {PY_VER: "36", PY_ARCH: "64"} + - {PY_VER: "35", PY_ARCH: "32"} + - {PY_VER: "35", PY_ARCH: "64"} + - {PY_VER: "34", PY_ARCH: "32"} + - {PY_VER: "34", PY_ARCH: "64"} OPENSSL_VERSION: "1_1_1b" POSTGRES_VERSION: "11_2" diff --git a/scripts/appveyor.py b/scripts/appveyor.py index abd68ed2..1a4f3f37 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -42,45 +42,44 @@ def setup_build_env(): """ Set the environment variables according to the build environment """ - setenv('VS_VER', vs_ver()) + setenv('VS_VER', opt.vs_ver) - if vs_ver() == '10.0' and opt.arch_64: + if opt.vs_ver == '10.0' and opt.arch_64: setenv('DISTUTILS_USE_SDK', '1') path = [ - str(py_dir()), - str(py_dir() / 'Scripts'), + str(opt.py_dir), + str(opt.py_dir / 'Scripts'), r'C:\Program Files\Git\mingw64\bin', os.environ['PATH'], ] setenv('PATH', os.pathsep.join(path)) - if vs_ver() == '9.0': + if opt.vs_ver == '9.0': logger.info("Fixing VS2008 Express and 64bit builds") shutil.copyfile( - vc_dir() / r"bin\vcvars64.bat", - vc_dir() / r"bin\amd64\vcvarsamd64.bat", + opt.vc_dir / "bin/vcvars64.bat", + opt.vc_dir / "bin/amd64/vcvarsamd64.bat", ) # Fix problem with VS2010 Express 64bit missing vcvars64.bat - if vs_ver() == '10.0': - if not (vc_dir() / r"bin\amd64\vcvars64.bat").exists(): + if opt.vs_ver == '10.0': + if not (opt.vc_dir / "bin/amd64/vcvars64.bat").exists(): logger.info("Fixing VS2010 Express and 64bit builds") copy_file( - package_dir() / r"scripts\vcvars64-vs2010.bat", - vc_dir() / r"bin\amd64\vcvars64.bat", + opt.package_dir / "scripts/vcvars64-vs2010.bat", + opt.vc_dir / "bin/amd64/vcvars64.bat", ) logger.info("Configuring compiler") - bat_call([vc_dir() / "vcvarsall.bat", 'x86' if opt.arch_32 else 'amd64']) + bat_call([opt.vc_dir / "vcvarsall.bat", 'x86' if opt.arch_32 else 'amd64']) def python_info(): logger.info("Python Information") - run_command([py_exe(), '--version'], stderr=sp.STDOUT) - run_command( - [py_exe(), '-c'] - + ["import sys; print('64bit: %s' % (sys.maxsize > 2**32))"] + run_python(['--version'], stderr=sp.STDOUT) + run_python( + ['-c', "import sys; print('64bit: %s' % (sys.maxsize > 2**32))"] ) @@ -89,7 +88,7 @@ def step_install(): configure_sdk() configure_postgres() - if is_wheel(): + if opt.is_wheel: install_wheel_support() @@ -97,8 +96,8 @@ def install_wheel_support(): """ Install an up-to-date pip wheel package to build wheels. """ - run_command([py_exe()] + "-m pip install --upgrade pip".split()) - run_command([py_exe()] + "-m pip install wheel".split()) + run_python("-m pip install --upgrade pip".split()) + run_python("-m pip install wheel".split()) def configure_sdk(): @@ -119,14 +118,14 @@ def configure_postgres(): Set up PostgreSQL config before the service starts. """ logger.info("Configuring Postgres") - with (pg_data_dir() / 'postgresql.conf').open('a') as f: + with (opt.pg_data_dir / 'postgresql.conf').open('a') as f: # allow > 1 prepared transactions for test cases print("max_prepared_transactions = 10", file=f) print("ssl = on", file=f) # Create openssl certificate to allow ssl connection cwd = os.getcwd() - os.chdir(pg_data_dir()) + os.chdir(opt.pg_data_dir) run_openssl( 'req -new -x509 -days 365 -nodes -text ' '-out server.crt -keyout server.key -subj /CN=initd.org'.split() @@ -167,12 +166,12 @@ def step_build_script(): build_libpq() build_psycopg() - if is_wheel(): + if opt.is_wheel: build_binary_packages() def build_openssl(): - top = ssl_build_dir() + top = opt.ssl_build_dir if (top / 'lib' / 'libssl.lib').exists(): return @@ -195,16 +194,16 @@ def build_openssl(): # Download OpenSSL source zipname = f'OpenSSL_{ver}.zip' - zipfile = cache_dir() / zipname + zipfile = opt.cache_dir / zipname if not zipfile.exists(): download( f"https://github.com/openssl/openssl/archive/{zipname}", zipfile ) with ZipFile(zipfile) as z: - z.extractall(path=build_dir()) + z.extractall(path=opt.build_dir) - sslbuild = build_dir() / f"openssl-OpenSSL_{ver}" + sslbuild = opt.build_dir / f"openssl-OpenSSL_{ver}" os.chdir(sslbuild) run_command( ['perl', 'Configure', target, 'no-asm'] @@ -215,12 +214,12 @@ def build_openssl(): assert (top / 'lib' / 'libssl.lib').exists() - os.chdir(clone_dir()) + os.chdir(opt.clone_dir) shutil.rmtree(sslbuild) def build_libpq(): - top = pg_build_dir() + top = opt.pg_build_dir if (top / 'lib' / 'libpq.lib').exists(): return @@ -235,7 +234,7 @@ def build_libpq(): # Download PostgreSQL source zipname = f'postgres-REL_{ver}.zip' - zipfile = cache_dir() / zipname + zipfile = opt.cache_dir / zipname if not zipfile.exists(): download( f"https://github.com/postgres/postgres/archive/REL_{ver}.zip", @@ -243,9 +242,9 @@ def build_libpq(): ) with ZipFile(zipfile) as z: - z.extractall(path=build_dir()) + z.extractall(path=opt.build_dir) - pgbuild = build_dir() / f"postgres-REL_{ver}" + pgbuild = opt.build_dir / f"postgres-REL_{ver}" os.chdir(pgbuild) # Patch for OpenSSL 1.1 configuration. See: @@ -272,7 +271,7 @@ $config->{openssl} = "%s"; 1; """ - % str(ssl_build_dir()).replace('\\', '\\\\'), + % str(opt.ssl_build_dir).replace('\\', '\\\\'), file=f, ) @@ -315,21 +314,21 @@ $config->{openssl} = "%s"; assert (top / 'lib' / 'libpq.lib').exists() assert (top / 'bin' / 'pg_config.exe').exists() - os.chdir(clone_dir()) + os.chdir(opt.clone_dir) shutil.rmtree(pgbuild) def build_psycopg(): - os.chdir(package_dir()) + os.chdir(opt.package_dir) patch_package_name() add_pg_config_path() - run_command( - [py_exe(), "setup.py", "build_ext", "--have-ssl"] + run_python( + ["setup.py", "build_ext", "--have-ssl"] + ["-l", "libpgcommon", "-l", "libpgport"] - + ["-L", ssl_build_dir() / 'lib'] - + ['-I', ssl_build_dir() / 'include'] + + ["-L", opt.ssl_build_dir / 'lib'] + + ['-I', opt.ssl_build_dir / 'include'] ) - run_command([py_exe(), "setup.py", "build_py"]) + run_python(["setup.py", "build_py"]) def patch_package_name(): @@ -340,7 +339,7 @@ def patch_package_name(): logger.info("changing package name to %s", conf) - with (package_dir() / 'setup.py').open() as f: + with (opt.package_dir / 'setup.py').open() as f: data = f.read() # Replace the name of the package with what desired @@ -348,26 +347,26 @@ def patch_package_name(): assert len(rex.findall(data)) == 1, rex.findall(data) data = rex.sub(f'name="{conf}"', data) - with (package_dir() / 'setup.py').open('w') as f: + with (opt.package_dir / 'setup.py').open('w') as f: f.write(data) def build_binary_packages(): """Create wheel/exe binary packages.""" - os.chdir(package_dir()) + os.chdir(opt.package_dir) add_pg_config_path() # Build .exe packages for whom still use them if os.environ['CONFIGURATION'] == 'psycopg2': - run_command([py_exe(), 'setup.py', 'bdist_wininst', "-d", dist_dir()]) + run_python(['setup.py', 'bdist_wininst', "-d", opt.dist_dir]) # Build .whl packages - run_command([py_exe(), 'setup.py', 'bdist_wheel', "-d", dist_dir()]) + run_python(['setup.py', 'bdist_wheel', "-d", opt.dist_dir]) def step_after_build(): - if not is_wheel(): + if not opt.is_wheel: install_built_package() else: install_binary_package() @@ -375,25 +374,25 @@ def step_after_build(): def install_built_package(): """Install the package just built by setup build.""" - os.chdir(package_dir()) + os.chdir(opt.package_dir) # Install the psycopg just built add_pg_config_path() - run_command([py_exe(), "setup.py", "install"]) + run_python(["setup.py", "install"]) shutil.rmtree("psycopg2.egg-info") def install_binary_package(): """Install the package from a packaged wheel.""" - run_command( - [py_exe(), '-m', 'pip', 'install', '--no-index', '-f', dist_dir()] + run_python( + ['-m', 'pip', 'install', '--no-index', '-f', opt.dist_dir] + [os.environ['CONFIGURATION']] ) def add_pg_config_path(): """Allow finding in the path the pg_config just built.""" - pg_path = str(pg_build_dir() / 'bin') + pg_path = str(opt.pg_build_dir / 'bin') if pg_path not in os.environ['PATH'].split(os.pathsep): setenv('PATH', os.pathsep.join([pg_path, os.environ['PATH']])) @@ -402,9 +401,9 @@ def step_before_test(): print_psycopg2_version() # Create and setup PostgreSQL database for the tests - run_command([pg_bin_dir() / 'createdb', os.environ['PSYCOPG2_TESTDB']]) + run_command([opt.pg_bin_dir / 'createdb', os.environ['PSYCOPG2_TESTDB']]) run_command( - [pg_bin_dir() / 'psql', '-d', os.environ['PSYCOPG2_TESTDB']] + [opt.pg_bin_dir / 'psql', '-d', os.environ['PSYCOPG2_TESTDB']] + ['-c', "CREATE EXTENSION hstore"] ) @@ -416,7 +415,7 @@ def print_psycopg2_version(): 'psycopg2.__libpq_version__', 'psycopg2.extensions.libpq_version()', ): - out = out_command([py_exe(), '-c', f"import psycopg2; print({expr})"]) + out = out_python(['-c', f"import psycopg2; print({expr})"]) logger.info("built %s: %s", expr, out.decode('ascii')) @@ -432,8 +431,8 @@ def check_libpq_version(): want_ver = tuple(map(int, os.environ['POSTGRES_VERSION'].split('_'))) want_ver = "%d%04d" % want_ver got_ver = ( - out_command( - [py_exe(), '-c'] + out_python( + ['-c'] + ["import psycopg2; print(psycopg2.extensions.libpq_version())"] ) .decode('ascii') @@ -450,19 +449,18 @@ def run_test_suite(): os.environ.pop('OPENSSL_CONF', None) # Run the unit test - cmdline = [ - py_exe(), + args = [ '-c', "import tests; tests.unittest.main(defaultTest='tests.test_suite')", ] - if is_wheel(): + if opt.is_wheel: os.environ['PSYCOPG2_TEST_FAST'] = '1' else: - cmdline.append('--verbose') + args.append('--verbose') - os.chdir(package_dir()) - run_command(cmdline) + os.chdir(opt.package_dir) + run_python(args) def step_on_success(): @@ -477,7 +475,7 @@ def print_sha1_hashes(): """ logger.info("artifacts SHA1 hashes:") - os.chdir(package_dir() / 'dist') + os.chdir(opt.package_dir / 'dist') run_command([which('sha1sum'), '-b', f'psycopg2-*/*']) @@ -500,7 +498,7 @@ def setup_ssh(): # Write SSH Private Key file from environment variable pkey = pkey.replace(' ', '\n') - with (clone_dir() / 'id_rsa').open('w') as f: + with (opt.clone_dir / 'id_rsa').open('w') as f: f.write( f"""\ -----BEGIN RSA PRIVATE KEY----- @@ -520,11 +518,11 @@ def upload_packages(): logger.info("uploading artifacts") ssh_cmd = r"C:\MinGW\msys\1.0\bin\ssh -i %s -o UserKnownHostsFile=%s" % ( - clone_dir() / "id_rsa", - clone_dir() / 'known_hosts', + opt.clone_dir / "id_rsa", + opt.clone_dir / 'known_hosts', ) - os.chdir(package_dir()) + os.chdir(opt.package_dir) run_command( [r"C:\MinGW\msys\1.0\bin\rsync", "-avr"] + ["-e", ssh_cmd, "dist/", "upload@initd.org:"] @@ -588,11 +586,10 @@ def bat_call(cmdline): cmdline = map(str, cmdline) cmdline = ' '.join(c if ' ' not in c else '"%s"' % c for c in cmdline) - pyexe = py_exe() - data = f"""\ CALL {cmdline} -{pyexe} -c "import os, sys, json; json.dump(dict(os.environ), sys.stdout, indent=2)" +{opt.py_exe} -c "import os, sys, json; \ +json.dump(dict(os.environ), sys.stdout, indent=2)" """ logger.debug("preparing file to batcall:\n\n%s", data) @@ -622,126 +619,6 @@ CALL {cmdline} os.remove(fn) -def py_dir(): - """ - Return the path to the target python binary to execute. - """ - dirname = ''.join([r"C:\Python", opt.pyver, '-x64' if opt.arch_64 else '']) - return Path(dirname) - - -def py_exe(): - """ - Return the full path of the target python executable. - """ - return py_dir() / 'python.exe' - - -def vc_dir(vsver=None): - """ - Return the path of the Visual C compiler. - """ - if vsver is None: - vsver = vs_ver() - - return Path( - r"C:\Program Files (x86)\Microsoft Visual Studio %s\VC" % vsver - ) - - -def vs_ver(pyver=None): - # Py 2.7 = VS Ver. 9.0 (VS 2008) - # Py 3.4 = VS Ver. 10.0 (VS 2010) - # Py 3.5, 3.6, 3.7 = VS Ver. 14.0 (VS 2015) - if pyver is None: - pyver = opt.pyver - - if pyver == '27': - vsver = '9.0' - elif pyver == '34': - vsver = '10.0' - elif pyver in ('35', '36', '37'): - vsver = '14.0' - else: - raise Exception('unexpected python version: %r' % pyver) - - return vsver - - -def clone_dir(): - return Path(r"C:\Project") - - -def appveyor_pg_dir(): - return Path(os.environ['POSTGRES_DIR']) - - -def pg_data_dir(): - return appveyor_pg_dir() / 'data' - - -def pg_bin_dir(): - return appveyor_pg_dir() / 'bin' - - -def pg_build_dir(): - return cache_arch_dir() / 'postgresql' - - -def ssl_build_dir(): - return cache_arch_dir() / 'openssl' - - -def cache_arch_dir(): - rv = cache_dir() / opt.pyarch / vs_ver() - return ensure_dir(rv) - - -def cache_dir(): - return Path(r"C:\Others") - - -def build_dir(): - rv = cache_arch_dir() / 'Builds' - return ensure_dir(rv) - - -def package_dir(): - """ - Return the directory containing the psycopg code checkout dir - - Building psycopg is clone_dir(), building the wheel packages is a submodule. - """ - return clone_dir() / 'psycopg2' if is_wheel() else clone_dir() - - -def is_wheel(): - """ - Return whether we are building the wheel packages or just the extension. - """ - project_name = os.environ['APPVEYOR_PROJECT_NAME'] - if project_name == 'psycopg2': - return False - elif project_name == 'psycopg2-wheels': - return True - else: - raise Exception(f"unexpected project name: {project_name}") - - -def dist_dir(): - return package_dir() / 'dist' / ('psycopg2-%s' % package_version()) - - -def package_version(): - with (package_dir() / 'setup.py').open() as f: - data = f.read() - - m = re.search( - r"""^PSYCOPG_VERSION\s*=\s*['"](.*)['"]""", data, re.MULTILINE - ) - return m.group(1) - - def ensure_dir(dir): if not isinstance(dir, Path): dir = Path(dir) @@ -754,6 +631,7 @@ def ensure_dir(dir): def run_command(cmdline, **kwargs): + """Run a command, raise on error.""" if not isinstance(cmdline, str): cmdline = list(map(str, cmdline)) logger.info("running command: %s", cmdline) @@ -761,6 +639,7 @@ def run_command(cmdline, **kwargs): def out_command(cmdline, **kwargs): + """Run a command, return its output, raise on error.""" if not isinstance(cmdline, str): cmdline = list(map(str, cmdline)) logger.info("running command: %s", cmdline) @@ -768,6 +647,20 @@ def out_command(cmdline, **kwargs): return data +def run_python(args, **kwargs): + """ + Run a script in the target Python. + """ + return run_command([opt.py_exe] + args, **kwargs) + + +def out_python(args, **kwargs): + """ + Return the output of a script run in the target Python. + """ + return out_command([opt.py_exe] + args, **kwargs) + + def copy_file(src, dst): logger.info("copying file %s -> %s", src, dst) shutil.copy(src, dst) @@ -797,23 +690,165 @@ def which(name): raise Exception("couldn't find program on path: %s" % name) +class Options: + """ + An object exposing the script configuration from env vars and command line. + """ + + @property + def py_ver(self): + """The Python version to build as 2 digits string.""" + rv = os.environ['PY_VER'] + assert rv in ('27', '34', '35', '36', '37'), rv + return rv + + @property + def py_arch(self): + """The Python architecture to build, 32 or 64.""" + rv = os.environ['PY_ARCH'] + assert rv in ('32', '64'), rv + return int(rv) + + @property + def arch_32(self): + """True if the Python architecture to build is 32 bits.""" + return self.py_arch == 32 + + @property + def arch_64(self): + """True if the Python architecture to build is 64 bits.""" + return self.py_arch == 64 + + @property + def package_version(self): + """The psycopg2 version number to build.""" + with (self.package_dir / 'setup.py').open() as f: + data = f.read() + + m = re.search( + r"""^PSYCOPG_VERSION\s*=\s*['"](.*)['"]""", data, re.MULTILINE + ) + return m.group(1) + + @property + def is_wheel(self): + """Are we building the wheel packages or just the extension?""" + project_name = os.environ['APPVEYOR_PROJECT_NAME'] + if project_name == 'psycopg2': + return False + elif project_name == 'psycopg2-wheels': + return True + else: + raise Exception(f"unexpected project name: {project_name}") + + @property + def py_dir(self): + """ + The path to the target python binary to execute. + """ + dirname = ''.join( + [r"C:\Python", self.py_ver, '-x64' if self.arch_64 else ''] + ) + return Path(dirname) + + @property + def py_exe(self): + """ + The full path of the target python executable. + """ + return self.py_dir / 'python.exe' + + @property + def vc_dir(self): + """ + The path of the Visual C compiler. + """ + return Path( + r"C:\Program Files (x86)\Microsoft Visual Studio %s\VC" + % self.vs_ver + ) + + @property + def vs_ver(self): + # Py 2.7 = VS Ver. 9.0 (VS 2008) + # Py 3.3, 3.4 = VS Ver. 10.0 (VS 2010) + # Py 3.5, 3.6, 3.7 = VS Ver. 14.0 (VS 2015) + vsvers = { + '27': '9.0', + '33': '10.0', + '34': '10.0', + '35': '14.0', + '36': '14.0', + '37': '14.0', + } + return vsvers[self.py_ver] + + @property + def clone_dir(self): + """The directory where the repository is cloned.""" + return Path(r"C:\Project") + + @property + def appveyor_pg_dir(self): + """The directory of the postgres service made available by Appveyor.""" + return Path(os.environ['POSTGRES_DIR']) + + @property + def pg_data_dir(self): + """The data dir of the appveyor postgres service.""" + return self.appveyor_pg_dir / 'data' + + @property + def pg_bin_dir(self): + """The bin dir of the appveyor postgres service.""" + return self.appveyor_pg_dir / 'bin' + + @property + def pg_build_dir(self): + """The directory where to build the postgres libraries for psycopg.""" + return self.cache_arch_dir / 'postgresql' + + @property + def ssl_build_dir(self): + """The directory where to build the openssl libraries for psycopg.""" + return self.cache_arch_dir / 'openssl' + + @property + def cache_arch_dir(self): + rv = self.cache_dir / str(self.py_arch) / self.vs_ver + return ensure_dir(rv) + + @property + def cache_dir(self): + return Path(r"C:\Others") + + @property + def build_dir(self): + rv = self.cache_arch_dir / 'Builds' + return ensure_dir(rv) + + @property + def package_dir(self): + """ + The directory containing the psycopg code checkout dir. + + Building psycopg it is clone_dir, building the wheels it is a submodule. + """ + return self.clone_dir / 'psycopg2' if self.is_wheel else self.clone_dir + + @property + def dist_dir(self): + """The directory where to build packages to distribute.""" + return ( + self.package_dir / 'dist' / ('psycopg2-%s' % self.package_version) + ) + + def parse_cmdline(): from argparse import ArgumentParser parser = ArgumentParser(description=__doc__) - parser.add_argument( - '--pyver', - choices='27 34 35 36 37'.split(), - help="the target python version. Default from PYVER env var", - ) - - parser.add_argument( - '--pyarch', - choices='32 64'.split(), - help="the target python architecture. Default from PYTHON_ARCH env var", - ) - g = parser.add_mutually_exclusive_group() g.add_argument( '-q', @@ -844,17 +879,7 @@ def parse_cmdline(): 'step', choices=steps, help="the appveyor step to execute" ) - opt = parser.parse_args() - - # And die if they are not there. - if not opt.pyver: - opt.pyver = os.environ['PYVER'] - if not opt.pyarch: - opt.pyarch = os.environ['PYTHON_ARCH'] - assert opt.pyarch in ('32', '64') - - opt.arch_32 = opt.pyarch == '32' - opt.arch_64 = opt.pyarch == '64' + opt = parser.parse_args(namespace=Options()) return opt From ed7d8ea28cfc908a7ce9f77014417db4dfce6da5 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 22 Apr 2019 22:39:21 +0100 Subject: [PATCH 17/17] Appveyor: added package_name to options --- scripts/appveyor.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/appveyor.py b/scripts/appveyor.py index 1a4f3f37..f5d8c46b 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -333,11 +333,10 @@ def build_psycopg(): def patch_package_name(): """Change the psycopg2 package name in the setup.py if required.""" - conf = os.environ.get('CONFIGURATION', 'psycopg2') - if conf == 'psycopg2': + if opt.package_name == 'psycopg2': return - logger.info("changing package name to %s", conf) + logger.info("changing package name to %s", opt.package_name) with (opt.package_dir / 'setup.py').open() as f: data = f.read() @@ -345,7 +344,7 @@ def patch_package_name(): # Replace the name of the package with what desired rex = re.compile(r"""name=["']psycopg2["']""") assert len(rex.findall(data)) == 1, rex.findall(data) - data = rex.sub(f'name="{conf}"', data) + data = rex.sub(f'name="{opt.package_name}"', data) with (opt.package_dir / 'setup.py').open('w') as f: f.write(data) @@ -358,7 +357,7 @@ def build_binary_packages(): add_pg_config_path() # Build .exe packages for whom still use them - if os.environ['CONFIGURATION'] == 'psycopg2': + if opt.package_name == 'psycopg2': run_python(['setup.py', 'bdist_wininst', "-d", opt.dist_dir]) # Build .whl packages @@ -386,7 +385,7 @@ def install_binary_package(): """Install the package from a packaged wheel.""" run_python( ['-m', 'pip', 'install', '--no-index', '-f', opt.dist_dir] - + [os.environ['CONFIGURATION']] + + [opt.package_name] ) @@ -719,6 +718,10 @@ class Options: """True if the Python architecture to build is 64 bits.""" return self.py_arch == 64 + @property + def package_name(self): + return os.environ.get('CONFIGURATION', 'psycopg2') + @property def package_version(self): """The psycopg2 version number to build."""