diff --git a/.appveyor.yml b/.appveyor.yml index 81515e40..13d4e251 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,42 +6,44 @@ environment: global: # MSVC Express 2008's setenv.cmd failes if /E:ON and /V:ON are not # enabled in the batch script interpreter - # - # WITH_COMPILER: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_compiler.cmd" CMD_IN_ENV: cmd /E:ON /V:ON /C .\appveyor\run_with_env.cmd matrix: # For Python versions available on Appveyor, see # https://www.appveyor.com/docs/build-environment/ - - {PYVER: "27", PYTHON_ARCH: "32"} - - {PYVER: "27", PYTHON_ARCH: "64"} - - {PYVER: "37", PYTHON_ARCH: "32"} - - {PYVER: "37", PYTHON_ARCH: "64"} - - {PYVER: "36", PYTHON_ARCH: "32"} - - {PYVER: "36", PYTHON_ARCH: "64"} - - {PYVER: "35", PYTHON_ARCH: "32"} - - {PYVER: "35", PYTHON_ARCH: "64"} - - {PYVER: "34", PYTHON_ARCH: "32"} - - {PYVER: "34", PYTHON_ARCH: "64"} + - {PY_VER: "27", PY_ARCH: "32"} + - {PY_VER: "27", PY_ARCH: "64"} + - {PY_VER: "37", PY_ARCH: "32"} + - {PY_VER: "37", PY_ARCH: "64"} + - {PY_VER: "36", PY_ARCH: "32"} + - {PY_VER: "36", PY_ARCH: "64"} + - {PY_VER: "35", PY_ARCH: "32"} + - {PY_VER: "35", PY_ARCH: "64"} + - {PY_VER: "34", PY_ARCH: "32"} + - {PY_VER: "34", PY_ARCH: "64"} OPENSSL_VERSION: "1_1_1b" POSTGRES_VERSION: "11_2" PSYCOPG2_TESTDB: psycopg2_test PSYCOPG2_TESTDB_USER: postgres - PSYCOPG2_TESTDB_PASSWORD: Password12! PSYCOPG2_TESTDB_HOST: localhost - PSYCOPG2_TESTDB_PORT: 5432 PGUSER: postgres PGPASSWORD: Password12! + PGSSLMODE: require + + # Select according to the service enabled + POSTGRES_DIR: C:\Program Files\PostgreSQL\9.6\ + + # The python used in the build process, not the one packages are built for + PYEXE: C:\Python36\python.exe matrix: fast_finish: false services: - # Note: if you change this service also change the paths to match - # (see where Program Files\Postgres\9.6 is used) + # Note: if you change this service also change POSTGRES_DIR - postgresql96 cache: @@ -50,189 +52,28 @@ cache: - C:\Others -> scripts\appveyor.cache_rebuild # Script called before repo cloning -init: - # Uncomment next line to get RDP access during the build. - #- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - - # Set env variable according to the build environment - - SET PYTHON=C:\Python%PYVER% - - IF "%PYTHON_ARCH%"=="64" SET PYTHON=%PYTHON%-x64 - - # Py 2.7 = VS Ver. 9.0 (VS 2008) - # Py 3.4 = VS Ver. 10.0 (VS 2010) - # Py 3.5, 3.6, 3.7 = VS Ver. 14.0 (VS 2015) - - IF "%PYVER%"=="27" SET VS_VER=9.0 - - IF "%PYVER%"=="34" SET VS_VER=10.0 - - IF "%PYVER%"=="35" SET VS_VER=14.0 - - IF "%PYVER%"=="36" SET VS_VER=14.0 - - IF "%PYVER%"=="37" SET VS_VER=14.0 - - - IF "%VS_VER%"=="10.0" IF "%PYTHON_ARCH%"=="64" SET DISTUTILS_USE_SDK=1 - - # Set Python to the path - - SET PATH=%PYTHON%;%PYTHON%\Scripts;C:\Program Files\Git\mingw64\bin;%PATH% - - # Verify Python version and architecture - - ECHO ******************************************************************* - - ECHO Python Information - - ECHO ******************************************************************* - - "%PYTHON%\\python --version" - - "%PYTHON%\\python -c \"import sys; print('64bit: ' + str(sys.maxsize > 2**32))\"" - - # Get & Install NASM - #- curl -L -o nasminst.exe https://www.nasm.us/pub/nasm/releasebuilds/2.12.02/win64/nasm-2.12.02-installer-x64.exe && start /wait nasminst.exe /S - #- SET PATH="C:\Program Files (x86)\nasm;%PATH%" - - # Fix problem with VS2008 Express and 64bit builds - - ECHO Fixing VS2008 Express and 64bit builds - - COPY "C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\bin\\vcvars64.bat" "C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\bin\\amd64\\vcvarsamd64.bat" - - # Fix problem with VS2010 Express 64bit missing vcvars64.bat - # Note: repository not cloned at this point, so need to fetch - # file another way - - ECHO Fixing VS2010 Express and 64bit builds - - curl -fsSL -o "C:\\Program Files (x86)\\Microsoft Visual Studio 10.0\\VC\\bin\\amd64\\vcvars64.bat" https://raw.githubusercontent.com/psycopg/psycopg2/master/scripts/vcvars64-vs2010.bat - - # Setup the compiler based upon version and architecture - - ECHO Configuring Compiler - - IF "%PYTHON_ARCH%"=="32" (CALL "C:\\Program Files (x86)\\Microsoft Visual Studio %VS_VER%\\VC\\vcvarsall.bat" x86) - - IF "%PYTHON_ARCH%"=="64" (CALL "C:\\Program Files (x86)\\Microsoft Visual Studio %VS_VER%\\VC\\vcvarsall.bat" amd64) - - # The program rc.exe on 64bit with some versions look in the wrong path - # location when building postgresql. This cheats by copying the x64 bit - # files to that location. - - IF "%PYTHON_ARCH%"=="64" (COPY /Y "C:\\Program Files\\Microsoft SDKs\\Windows\\v7.0\\Bin\\x64\\rc*" "C:\\Program Files (x86)\\Microsoft SDKs\\Windows\\v7.0A\\Bin") - - # Change PostgreSQL config before service starts to allow > 1 prepared - # transactions for test cases - - ECHO max_prepared_transactions = 10 >> "C:\\Program Files\\PostgreSQL\\9.6\\data\\postgresql.conf" - +# init: # Repository gets cloned, Cache is restored + install: - # We start off CD'ed to cloned folder - - SET BASE_DIR=C:\Others\%PYTHON_ARCH%\%VS_VER% - - SET BUILD_DIR=%BASE_DIR%\Builds - - IF NOT EXIST %BUILD_DIR% MKDIR %BUILD_DIR% + - "%PYEXE% scripts\\appveyor.py install" - - ECHO ******************************************************************* - - ECHO Initialized variables specific for this build - - ECHO ******************************************************************* - - ECHO %BASE_DIR% - - ECHO %BUILD_DIR% - - ECHO ******************************************************************* - - # Setup directories for building OpenSSL libraries - - ECHO ******************************************************************* - - ECHO Preparing for building OpenSSL - - ECHO ******************************************************************* - - SET OPENSSLTOP=%BASE_DIR%\openssl - - IF NOT EXIST %OPENSSLTOP%\include\openssl MKDIR %OPENSSLTOP%\include\openssl - - IF NOT EXIST %OPENSSLTOP%\lib MKDIR %OPENSSLTOP%\lib - - # Setup OpenSSL Environment Variables based on processor architecture - - ps: >- - If ($env:PYTHON_ARCH -Match "32" ) { - $env:VCVARS_PLATFORM="x86" - $env:TARGET="VC-WIN32" - } Else { - $env:VCVARS_PLATFORM="amd64" - $env:TARGET="VC-WIN64A" - $env:CPU="AMD64" - } - # Download OpenSSL source - - CD C:\Others - - IF NOT EXIST OpenSSL_%OPENSSL_VERSION%.zip ( - curl -fsSL -o OpenSSL_%OPENSSL_VERSION%.zip https://github.com/openssl/openssl/archive/OpenSSL_%OPENSSL_VERSION%.zip - ) - - - IF NOT EXIST %OPENSSLTOP%\lib\libssl.lib ( - CD %BUILD_DIR% && - 7z x C:\Others\OpenSSL_%OPENSSL_VERSION%.zip && - CD openssl-OpenSSL_%OPENSSL_VERSION% && - perl Configure %TARGET% no-asm no-shared no-zlib --prefix=%OPENSSLTOP% --openssldir=%OPENSSLTOP% && - nmake build_libs install_dev && - CD %BASE_DIR% && - RMDIR /S /Q %BUILD_DIR%\openssl-OpenSSL_%OPENSSL_VERSION% - ) - - # Setup directories for building PostgreSQL librarires - - ECHO ******************************************************************* - - ECHO Preparing for building PostgreSQL libraries - - ECHO ******************************************************************* - - SET PGTOP=%BASE_DIR%\postgresql - - IF NOT EXIST %PGTOP%\include MKDIR %PGTOP%\include - - IF NOT EXIST %PGTOP%\lib MKDIR %PGTOP%\lib - - IF NOT EXIST %PGTOP%\bin MKDIR %PGTOP%\bin - - # Download PostgreSQL source - - CD C:\Others - - IF NOT EXIST postgres-REL_%POSTGRES_VERSION%.zip ( - curl -fsSL -o postgres-REL_%POSTGRES_VERSION%.zip https://github.com/postgres/postgres/archive/REL_%POSTGRES_VERSION%.zip - ) - - # Setup build config file (config.pl) - # Hack the Mkvcbuild.pm file so we build the lib version of libpq - # Build libpgport, libpgcommon, libpq - # Install includes - # Copy over built libraries - # Prepare local include directory for building from - # Build pg_config in place - # Note patch for OpenSSL 1.1 configuration. See: - # https://www.postgresql-archive.org/Compile-psql-9-6-with-SSL-Version-1-1-0-td6054118.html - # NOTE: Cannot set and use the same variable inside an IF - - SET PGBUILD=%BUILD_DIR%\postgres-REL_%POSTGRES_VERSION% - - IF NOT EXIST %PGTOP%\lib\libpq.lib ( - CD %BUILD_DIR% && - 7z x C:\Others\postgres-REL_%POSTGRES_VERSION%.zip && - CD postgres-REL_%POSTGRES_VERSION% && - patch -p1 < %APPVEYOR_BUILD_FOLDER%\scripts\win_openssl_11.patch && - CD src\tools\msvc && - ECHO $config-^>{ldap} = 0; > config.pl && - ECHO $config-^>{openssl} = "%OPENSSLTOP:\=\\%"; >> config.pl && - ECHO.>> config.pl && - ECHO 1;>> config.pl && - perl -pi.bak -e "s/'libpq', 'dll'/'libpq', 'lib'/g" Mkvcbuild.pm && - build libpgport && - build libpgcommon && - build libpq && - ECHO "" > %PGBUILD%\src\backend\parser\gram.h && - perl -pi.bak -e "s/qw\(Install\)/qw\(Install CopyIncludeFiles\)/g" Install.pm && - perl -MInstall=CopyIncludeFiles -e"chdir('../../..'); CopyIncludeFiles('%PGTOP%')" && - COPY %PGBUILD%\Release\libpgport\libpgport.lib %PGTOP%\lib && - COPY %PGBUILD%\Release\libpgcommon\libpgcommon.lib %PGTOP%\lib && - COPY %PGBUILD%\Release\libpq\libpq.lib %PGTOP%\lib && - XCOPY /Y /S %PGBUILD%\src\include\port\win32\* %PGBUILD%\src\include && - XCOPY /Y /S %PGBUILD%\src\include\port\win32_msvc\* %PGBUILD%\src\include && - CD %PGBUILD%\src\bin\pg_config && - cl pg_config.c /MT /nologo /I%PGBUILD%\src\include /link /LIBPATH:%PGTOP%\lib libpgcommon.lib libpgport.lib advapi32.lib /NODEFAULTLIB:libcmt.lib /OUT:%PGTOP%\bin\pg_config.exe && - CD %BASE_DIR% && - RMDIR /S /Q %PGBUILD% - ) +# PostgreSQL server starts now build: off -#before_build: - build_script: - # Add PostgreSQL binaries to the path - - PATH=C:\Program Files\PostgreSQL\9.6\bin\;%PATH% - - CD C:\Project - - "%PYTHON%\\python.exe setup.py build_ext --have-ssl --pg-config %PGTOP%\\bin\\pg_config.exe -l libpgcommon -l libpgport -L %OPENSSLTOP%\\lib -I %OPENSSLTOP%\\include" - - "%PYTHON%\\python.exe setup.py build" - - "%PYTHON%\\python.exe setup.py install" - - RD /S /Q psycopg2.egg-info + - "%PYEXE% scripts\\appveyor.py build_script" -#after_build: +after_build: + - "%PYEXE% scripts\\appveyor.py after_build" before_test: - # Create and setup PostgreSQL database for the tests - - createdb %PSYCOPG2_TESTDB% - - psql -d %PSYCOPG2_TESTDB% -c "CREATE EXTENSION HSTORE;" + - "%PYEXE% scripts\\appveyor.py before_test" test_script: - # Print psycopg and libpq versions - - "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.__version__)\"" - - "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.__libpq_version__)\"" - - "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.extensions.libpq_version())\"" - - "%PYTHON%\\python.exe -c \"import tests; tests.unittest.main(defaultTest='tests.test_suite')\" --verbose" + - "%PYEXE% scripts\\appveyor.py test_script" + + +# vim: set ts=4 sts=4 sw=4: diff --git a/scripts/appveyor.py b/scripts/appveyor.py new file mode 100755 index 00000000..f5d8c46b --- /dev/null +++ b/scripts/appveyor.py @@ -0,0 +1,891 @@ +#!/usr/bin/env python3 +""" +Build steps for the windows binary packages. + +The script is designed to be called by appveyor. Subcommands map the steps in +'appveyor.yml'. + +""" + +import re +import os +import sys +import json +import shutil +import logging +import subprocess as sp +from glob import glob +from pathlib import Path +from zipfile import ZipFile +from tempfile import NamedTemporaryFile +from urllib.request import urlopen + +opt = None +STEP_PREFIX = 'step_' + +logger = logging.getLogger() +logging.basicConfig( + level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s' +) + + +def main(): + global opt + opt = parse_cmdline() + logger.setLevel(opt.loglevel) + + cmd = globals()[STEP_PREFIX + opt.step] + cmd() + + +def setup_build_env(): + """ + Set the environment variables according to the build environment + """ + setenv('VS_VER', opt.vs_ver) + + if opt.vs_ver == '10.0' and opt.arch_64: + setenv('DISTUTILS_USE_SDK', '1') + + path = [ + str(opt.py_dir), + str(opt.py_dir / 'Scripts'), + r'C:\Program Files\Git\mingw64\bin', + os.environ['PATH'], + ] + setenv('PATH', os.pathsep.join(path)) + + if opt.vs_ver == '9.0': + logger.info("Fixing VS2008 Express and 64bit builds") + shutil.copyfile( + opt.vc_dir / "bin/vcvars64.bat", + opt.vc_dir / "bin/amd64/vcvarsamd64.bat", + ) + + # Fix problem with VS2010 Express 64bit missing vcvars64.bat + if opt.vs_ver == '10.0': + if not (opt.vc_dir / "bin/amd64/vcvars64.bat").exists(): + logger.info("Fixing VS2010 Express and 64bit builds") + copy_file( + opt.package_dir / "scripts/vcvars64-vs2010.bat", + opt.vc_dir / "bin/amd64/vcvars64.bat", + ) + + logger.info("Configuring compiler") + bat_call([opt.vc_dir / "vcvarsall.bat", 'x86' if opt.arch_32 else 'amd64']) + + +def python_info(): + logger.info("Python Information") + run_python(['--version'], stderr=sp.STDOUT) + run_python( + ['-c', "import sys; print('64bit: %s' % (sys.maxsize > 2**32))"] + ) + + +def step_install(): + python_info() + configure_sdk() + configure_postgres() + + if opt.is_wheel: + install_wheel_support() + + +def install_wheel_support(): + """ + Install an up-to-date pip wheel package to build wheels. + """ + run_python("-m pip install --upgrade pip".split()) + run_python("-m pip install wheel".split()) + + +def configure_sdk(): + # The program rc.exe on 64bit with some versions look in the wrong path + # location when building postgresql. This cheats by copying the x64 bit + # files to that location. + if opt.arch_64: + for fn in glob( + r'C:\Program Files\Microsoft SDKs\Windows\v7.0\Bin\x64\rc*' + ): + copy_file( + fn, r"C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Bin" + ) + + +def configure_postgres(): + """ + Set up PostgreSQL config before the service starts. + """ + logger.info("Configuring Postgres") + with (opt.pg_data_dir / 'postgresql.conf').open('a') as f: + # allow > 1 prepared transactions for test cases + print("max_prepared_transactions = 10", file=f) + print("ssl = on", file=f) + + # Create openssl certificate to allow ssl connection + cwd = os.getcwd() + os.chdir(opt.pg_data_dir) + run_openssl( + 'req -new -x509 -days 365 -nodes -text ' + '-out server.crt -keyout server.key -subj /CN=initd.org'.split() + ) + run_openssl( + 'req -new -nodes -text -out root.csr -keyout root.key ' + '-subj /CN=initd.org'.split() + ) + + run_openssl( + 'x509 -req -in root.csr -text -days 3650 -extensions v3_ca ' + '-signkey root.key -out root.crt'.split() + ) + + run_openssl( + 'req -new -nodes -text -out server.csr -keyout server.key ' + '-subj /CN=initd.org'.split() + ) + + run_openssl( + 'x509 -req -in server.csr -text -days 365 -CA root.crt ' + '-CAkey root.key -CAcreateserial -out server.crt'.split() + ) + + os.chdir(cwd) + + +def run_openssl(args): + """Run the appveyor-installed openssl with some args.""" + # https://www.appveyor.com/docs/windows-images-software/ + openssl = Path(r"C:\OpenSSL-v111-Win64") / 'bin' / 'openssl' + return run_command([openssl] + args) + + +def step_build_script(): + setup_build_env() + build_openssl() + build_libpq() + build_psycopg() + + if opt.is_wheel: + build_binary_packages() + + +def build_openssl(): + top = opt.ssl_build_dir + if (top / 'lib' / 'libssl.lib').exists(): + return + + logger.info("Building OpenSSL") + + # Setup directories for building OpenSSL libraries + ensure_dir(top / 'include' / 'openssl') + ensure_dir(top / 'lib') + + # Setup OpenSSL Environment Variables based on processor architecture + if opt.arch_32: + target = 'VC-WIN32' + setenv('VCVARS_PLATFORM', 'x86') + else: + target = 'VC-WIN64A' + setenv('VCVARS_PLATFORM', 'amd64') + setenv('CPU', 'AMD64') + + ver = os.environ['OPENSSL_VERSION'] + + # Download OpenSSL source + zipname = f'OpenSSL_{ver}.zip' + zipfile = opt.cache_dir / zipname + if not zipfile.exists(): + download( + f"https://github.com/openssl/openssl/archive/{zipname}", zipfile + ) + + with ZipFile(zipfile) as z: + z.extractall(path=opt.build_dir) + + sslbuild = opt.build_dir / f"openssl-OpenSSL_{ver}" + os.chdir(sslbuild) + run_command( + ['perl', 'Configure', target, 'no-asm'] + + ['no-shared', 'no-zlib', f'--prefix={top}', f'--openssldir={top}'] + ) + + run_command("nmake build_libs install_dev".split()) + + assert (top / 'lib' / 'libssl.lib').exists() + + os.chdir(opt.clone_dir) + shutil.rmtree(sslbuild) + + +def build_libpq(): + top = opt.pg_build_dir + if (top / 'lib' / 'libpq.lib').exists(): + return + + logger.info("Building libpq") + + # Setup directories for building PostgreSQL librarires + ensure_dir(top / 'include') + ensure_dir(top / 'lib') + ensure_dir(top / 'bin') + + ver = os.environ['POSTGRES_VERSION'] + + # Download PostgreSQL source + zipname = f'postgres-REL_{ver}.zip' + zipfile = opt.cache_dir / zipname + if not zipfile.exists(): + download( + f"https://github.com/postgres/postgres/archive/REL_{ver}.zip", + zipfile, + ) + + with ZipFile(zipfile) as z: + z.extractall(path=opt.build_dir) + + pgbuild = opt.build_dir / f"postgres-REL_{ver}" + os.chdir(pgbuild) + + # Patch for OpenSSL 1.1 configuration. See: + # https://www.postgresql-archive.org/Compile-psql-9-6-with-SSL-Version-1-1-0-td6054118.html + assert Path("src/include/pg_config.h.win32").exists() + with open("src/include/pg_config.h.win32", 'a') as f: + print( + """ +#define HAVE_ASN1_STRING_GET0_DATA 1 +#define HAVE_BIO_GET_DATA 1 +#define HAVE_BIO_METH_NEW 1 +#define HAVE_OPENSSL_INIT_SSL 1 +""", + file=f, + ) + + # Setup build config file (config.pl) + os.chdir("src/tools/msvc") + with open("config.pl", 'w') as f: + print( + """\ +$config->{ldap} = 0; +$config->{openssl} = "%s"; + +1; +""" + % str(opt.ssl_build_dir).replace('\\', '\\\\'), + file=f, + ) + + # Hack the Mkvcbuild.pm file so we build the lib version of libpq + file_replace('Mkvcbuild.pm', "'libpq', 'dll'", "'libpq', 'lib'") + + # Build libpgport, libpgcommon, libpq + run_command([which("build"), "libpgport"]) + run_command([which("build"), "libpgcommon"]) + run_command([which("build"), "libpq"]) + + # Install includes + with (pgbuild / "src/backend/parser/gram.h").open("w") as f: + print("", file=f) + + # Copy over built libraries + file_replace("Install.pm", "qw(Install)", "qw(Install CopyIncludeFiles)") + run_command( + ["perl", "-MInstall=CopyIncludeFiles", "-e"] + + [f"chdir('../../..'); CopyIncludeFiles('{top}')"] + ) + + for lib in ('libpgport', 'libpgcommon', 'libpq'): + copy_file(pgbuild / f'Release/{lib}/{lib}.lib', top / 'lib') + + # Prepare local include directory for building from + for dir in ('win32', 'win32_msvc'): + merge_dir(pgbuild / f"src/include/port/{dir}", pgbuild / "src/include") + + # Build pg_config in place + os.chdir(pgbuild / 'src/bin/pg_config') + run_command( + ['cl', 'pg_config.c', '/MT', '/nologo', fr'/I{pgbuild}\src\include'] + + ['/link', fr'/LIBPATH:{top}\lib'] + + ['libpgcommon.lib', 'libpgport.lib', 'advapi32.lib'] + + ['/NODEFAULTLIB:libcmt.lib'] + + [fr'/OUT:{top}\bin\pg_config.exe'] + ) + + assert (top / 'lib' / 'libpq.lib').exists() + assert (top / 'bin' / 'pg_config.exe').exists() + + os.chdir(opt.clone_dir) + shutil.rmtree(pgbuild) + + +def build_psycopg(): + os.chdir(opt.package_dir) + patch_package_name() + add_pg_config_path() + run_python( + ["setup.py", "build_ext", "--have-ssl"] + + ["-l", "libpgcommon", "-l", "libpgport"] + + ["-L", opt.ssl_build_dir / 'lib'] + + ['-I', opt.ssl_build_dir / 'include'] + ) + run_python(["setup.py", "build_py"]) + + +def patch_package_name(): + """Change the psycopg2 package name in the setup.py if required.""" + if opt.package_name == 'psycopg2': + return + + logger.info("changing package name to %s", opt.package_name) + + with (opt.package_dir / 'setup.py').open() as f: + data = f.read() + + # Replace the name of the package with what desired + rex = re.compile(r"""name=["']psycopg2["']""") + assert len(rex.findall(data)) == 1, rex.findall(data) + data = rex.sub(f'name="{opt.package_name}"', data) + + with (opt.package_dir / 'setup.py').open('w') as f: + f.write(data) + + +def build_binary_packages(): + """Create wheel/exe binary packages.""" + os.chdir(opt.package_dir) + + add_pg_config_path() + + # Build .exe packages for whom still use them + if opt.package_name == 'psycopg2': + run_python(['setup.py', 'bdist_wininst', "-d", opt.dist_dir]) + + # Build .whl packages + run_python(['setup.py', 'bdist_wheel', "-d", opt.dist_dir]) + + +def step_after_build(): + if not opt.is_wheel: + install_built_package() + else: + install_binary_package() + + +def install_built_package(): + """Install the package just built by setup build.""" + os.chdir(opt.package_dir) + + # Install the psycopg just built + add_pg_config_path() + run_python(["setup.py", "install"]) + shutil.rmtree("psycopg2.egg-info") + + +def install_binary_package(): + """Install the package from a packaged wheel.""" + run_python( + ['-m', 'pip', 'install', '--no-index', '-f', opt.dist_dir] + + [opt.package_name] + ) + + +def add_pg_config_path(): + """Allow finding in the path the pg_config just built.""" + pg_path = str(opt.pg_build_dir / 'bin') + if pg_path not in os.environ['PATH'].split(os.pathsep): + setenv('PATH', os.pathsep.join([pg_path, os.environ['PATH']])) + + +def step_before_test(): + print_psycopg2_version() + + # Create and setup PostgreSQL database for the tests + run_command([opt.pg_bin_dir / 'createdb', os.environ['PSYCOPG2_TESTDB']]) + run_command( + [opt.pg_bin_dir / 'psql', '-d', os.environ['PSYCOPG2_TESTDB']] + + ['-c', "CREATE EXTENSION hstore"] + ) + + +def print_psycopg2_version(): + """Print psycopg2 and libpq versions installed.""" + for expr in ( + 'psycopg2.__version__', + 'psycopg2.__libpq_version__', + 'psycopg2.extensions.libpq_version()', + ): + out = out_python(['-c', f"import psycopg2; print({expr})"]) + logger.info("built %s: %s", expr, out.decode('ascii')) + + +def step_test_script(): + check_libpq_version() + run_test_suite() + + +def check_libpq_version(): + """ + Fail if the package installed is not using the expected libpq version. + """ + want_ver = tuple(map(int, os.environ['POSTGRES_VERSION'].split('_'))) + want_ver = "%d%04d" % want_ver + got_ver = ( + out_python( + ['-c'] + + ["import psycopg2; print(psycopg2.extensions.libpq_version())"] + ) + .decode('ascii') + .rstrip() + ) + assert want_ver == got_ver, "libpq version mismatch: %r != %r" % ( + want_ver, + got_ver, + ) + + +def run_test_suite(): + # Remove this var, which would make badly a configured OpenSSL 1.1 work + os.environ.pop('OPENSSL_CONF', None) + + # Run the unit test + args = [ + '-c', + "import tests; tests.unittest.main(defaultTest='tests.test_suite')", + ] + + if opt.is_wheel: + os.environ['PSYCOPG2_TEST_FAST'] = '1' + else: + args.append('--verbose') + + os.chdir(opt.package_dir) + run_python(args) + + +def step_on_success(): + print_sha1_hashes() + if setup_ssh(): + upload_packages() + + +def print_sha1_hashes(): + """ + Print the packages sha1 so their integrity can be checked upon signing. + """ + logger.info("artifacts SHA1 hashes:") + + os.chdir(opt.package_dir / 'dist') + run_command([which('sha1sum'), '-b', f'psycopg2-*/*']) + + +def setup_ssh(): + """ + Configure ssh to upload built packages where they can be retrieved. + + Return False if can't configure and upload shoould be skipped. + """ + # If we are not on the psycopg AppVeyor account, the environment variable + # REMOTE_KEY will not be decrypted. In that case skip uploading. + if os.environ['APPVEYOR_ACCOUNT_NAME'] != 'psycopg': + logger.warn("skipping artifact upload: you are not psycopg") + return False + + pkey = os.environ.get('REMOTE_KEY', None) + if not pkey: + logger.warn("skipping artifact upload: no remote key") + return False + + # Write SSH Private Key file from environment variable + pkey = pkey.replace(' ', '\n') + with (opt.clone_dir / 'id_rsa').open('w') as f: + f.write( + f"""\ +-----BEGIN RSA PRIVATE KEY----- +{pkey} +-----END RSA PRIVATE KEY----- +""" + ) + + # Make a directory to please MinGW's version of ssh + ensure_dir(r"C:\MinGW\msys\1.0\home\appveyor\.ssh") + + return True + + +def upload_packages(): + # Upload built artifacts + logger.info("uploading artifacts") + + ssh_cmd = r"C:\MinGW\msys\1.0\bin\ssh -i %s -o UserKnownHostsFile=%s" % ( + opt.clone_dir / "id_rsa", + opt.clone_dir / 'known_hosts', + ) + + os.chdir(opt.package_dir) + run_command( + [r"C:\MinGW\msys\1.0\bin\rsync", "-avr"] + + ["-e", ssh_cmd, "dist/", "upload@initd.org:"] + ) + + +def download(url, fn): + """Download a file locally""" + logger.info("downloading %s", url) + with open(fn, 'wb') as fo, urlopen(url) as fi: + while 1: + data = fi.read(8192) + if not data: + break + fo.write(data) + + logger.info("file downloaded: %s", fn) + + +def file_replace(fn, s1, s2): + """ + Replace all the occurrences of the string s1 into s2 in the file fn. + """ + assert os.path.exists(fn) + with open(fn, 'r+') as f: + data = f.read() + f.seek(0) + f.write(data.replace(s1, s2)) + f.truncate() + + +def merge_dir(src, tgt): + """ + Merge the content of the directory src into the directory tgt + + Reproduce the semantic of "XCOPY /Y /S src/* tgt" + """ + src = str(src) + for dp, _dns, fns in os.walk(src): + logger.debug("dirpath %s", dp) + if not fns: + continue + assert dp.startswith(src) + subdir = dp[len(src) :].lstrip(os.sep) + tgtdir = ensure_dir(os.path.join(tgt, subdir)) + for fn in fns: + copy_file(os.path.join(dp, fn), tgtdir) + + +def bat_call(cmdline): + """ + Simulate 'CALL' from a batch file + + Execute CALL *cmdline* and export the changed environment to the current + environment. + + nana-nana-nana-nana... + + """ + if not isinstance(cmdline, str): + cmdline = map(str, cmdline) + cmdline = ' '.join(c if ' ' not in c else '"%s"' % c for c in cmdline) + + data = f"""\ +CALL {cmdline} +{opt.py_exe} -c "import os, sys, json; \ +json.dump(dict(os.environ), sys.stdout, indent=2)" +""" + + logger.debug("preparing file to batcall:\n\n%s", data) + + with NamedTemporaryFile(suffix='.bat') as tmp: + fn = tmp.name + + with open(fn, "w") as f: + f.write(data) + + try: + out = out_command(fn) + # be vewwy vewwy caweful to print the env var as it might contain + # secwet things like your pwecious pwivate key. + # logger.debug("output of command:\n\n%s", out.decode('utf8', 'replace')) + + # The output has some useless crap on stdout, because sure, and json + # indented so the last { on column 1 is where we have to start parsing + + m = list(re.finditer(b'^{', out, re.MULTILINE))[-1] + out = out[m.start() :] + env = json.loads(out) + for k, v in env.items(): + if os.environ.get(k) != v: + setenv(k, v) + finally: + os.remove(fn) + + +def ensure_dir(dir): + if not isinstance(dir, Path): + dir = Path(dir) + + if not dir.is_dir(): + logger.info("creating directory %s", dir) + dir.mkdir(parents=True) + + return dir + + +def run_command(cmdline, **kwargs): + """Run a command, raise on error.""" + if not isinstance(cmdline, str): + cmdline = list(map(str, cmdline)) + logger.info("running command: %s", cmdline) + sp.check_call(cmdline, **kwargs) + + +def out_command(cmdline, **kwargs): + """Run a command, return its output, raise on error.""" + if not isinstance(cmdline, str): + cmdline = list(map(str, cmdline)) + logger.info("running command: %s", cmdline) + data = sp.check_output(cmdline, **kwargs) + return data + + +def run_python(args, **kwargs): + """ + Run a script in the target Python. + """ + return run_command([opt.py_exe] + args, **kwargs) + + +def out_python(args, **kwargs): + """ + Return the output of a script run in the target Python. + """ + return out_command([opt.py_exe] + args, **kwargs) + + +def copy_file(src, dst): + logger.info("copying file %s -> %s", src, dst) + shutil.copy(src, dst) + + +def setenv(k, v): + logger.debug("setting %s=%s", k, v) + os.environ[k] = v + + +def which(name): + """ + Return the full path of a command found on the path + """ + base, ext = os.path.splitext(name) + if not ext: + exts = ('.com', '.exe', '.bat', '.cmd') + else: + exts = (ext,) + + for dir in ['.'] + os.environ['PATH'].split(os.pathsep): + for ext in exts: + fn = os.path.join(dir, base + ext) + if os.path.isfile(fn): + return fn + + raise Exception("couldn't find program on path: %s" % name) + + +class Options: + """ + An object exposing the script configuration from env vars and command line. + """ + + @property + def py_ver(self): + """The Python version to build as 2 digits string.""" + rv = os.environ['PY_VER'] + assert rv in ('27', '34', '35', '36', '37'), rv + return rv + + @property + def py_arch(self): + """The Python architecture to build, 32 or 64.""" + rv = os.environ['PY_ARCH'] + assert rv in ('32', '64'), rv + return int(rv) + + @property + def arch_32(self): + """True if the Python architecture to build is 32 bits.""" + return self.py_arch == 32 + + @property + def arch_64(self): + """True if the Python architecture to build is 64 bits.""" + return self.py_arch == 64 + + @property + def package_name(self): + return os.environ.get('CONFIGURATION', 'psycopg2') + + @property + def package_version(self): + """The psycopg2 version number to build.""" + with (self.package_dir / 'setup.py').open() as f: + data = f.read() + + m = re.search( + r"""^PSYCOPG_VERSION\s*=\s*['"](.*)['"]""", data, re.MULTILINE + ) + return m.group(1) + + @property + def is_wheel(self): + """Are we building the wheel packages or just the extension?""" + project_name = os.environ['APPVEYOR_PROJECT_NAME'] + if project_name == 'psycopg2': + return False + elif project_name == 'psycopg2-wheels': + return True + else: + raise Exception(f"unexpected project name: {project_name}") + + @property + def py_dir(self): + """ + The path to the target python binary to execute. + """ + dirname = ''.join( + [r"C:\Python", self.py_ver, '-x64' if self.arch_64 else ''] + ) + return Path(dirname) + + @property + def py_exe(self): + """ + The full path of the target python executable. + """ + return self.py_dir / 'python.exe' + + @property + def vc_dir(self): + """ + The path of the Visual C compiler. + """ + return Path( + r"C:\Program Files (x86)\Microsoft Visual Studio %s\VC" + % self.vs_ver + ) + + @property + def vs_ver(self): + # Py 2.7 = VS Ver. 9.0 (VS 2008) + # Py 3.3, 3.4 = VS Ver. 10.0 (VS 2010) + # Py 3.5, 3.6, 3.7 = VS Ver. 14.0 (VS 2015) + vsvers = { + '27': '9.0', + '33': '10.0', + '34': '10.0', + '35': '14.0', + '36': '14.0', + '37': '14.0', + } + return vsvers[self.py_ver] + + @property + def clone_dir(self): + """The directory where the repository is cloned.""" + return Path(r"C:\Project") + + @property + def appveyor_pg_dir(self): + """The directory of the postgres service made available by Appveyor.""" + return Path(os.environ['POSTGRES_DIR']) + + @property + def pg_data_dir(self): + """The data dir of the appveyor postgres service.""" + return self.appveyor_pg_dir / 'data' + + @property + def pg_bin_dir(self): + """The bin dir of the appveyor postgres service.""" + return self.appveyor_pg_dir / 'bin' + + @property + def pg_build_dir(self): + """The directory where to build the postgres libraries for psycopg.""" + return self.cache_arch_dir / 'postgresql' + + @property + def ssl_build_dir(self): + """The directory where to build the openssl libraries for psycopg.""" + return self.cache_arch_dir / 'openssl' + + @property + def cache_arch_dir(self): + rv = self.cache_dir / str(self.py_arch) / self.vs_ver + return ensure_dir(rv) + + @property + def cache_dir(self): + return Path(r"C:\Others") + + @property + def build_dir(self): + rv = self.cache_arch_dir / 'Builds' + return ensure_dir(rv) + + @property + def package_dir(self): + """ + The directory containing the psycopg code checkout dir. + + Building psycopg it is clone_dir, building the wheels it is a submodule. + """ + return self.clone_dir / 'psycopg2' if self.is_wheel else self.clone_dir + + @property + def dist_dir(self): + """The directory where to build packages to distribute.""" + return ( + self.package_dir / 'dist' / ('psycopg2-%s' % self.package_version) + ) + + +def parse_cmdline(): + from argparse import ArgumentParser + + parser = ArgumentParser(description=__doc__) + + g = parser.add_mutually_exclusive_group() + g.add_argument( + '-q', + '--quiet', + help="Talk less", + dest='loglevel', + action='store_const', + const=logging.WARN, + default=logging.INFO, + ) + g.add_argument( + '-v', + '--verbose', + help="Talk more", + dest='loglevel', + action='store_const', + const=logging.DEBUG, + default=logging.INFO, + ) + + steps = [ + n[len(STEP_PREFIX) :] + for n in globals() + if n.startswith(STEP_PREFIX) and callable(globals()[n]) + ] + + parser.add_argument( + 'step', choices=steps, help="the appveyor step to execute" + ) + + opt = parser.parse_args(namespace=Options()) + + return opt + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/win_openssl_11.patch b/scripts/win_openssl_11.patch deleted file mode 100644 index 57d0bd81..00000000 --- a/scripts/win_openssl_11.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/src/include/pg_config.h.win32 b/src/include/pg_config.h.win32 -index dfd6972383..72d1259ffb 100644 ---- a/src/include/pg_config.h.win32 -+++ b/src/include/pg_config.h.win32 -@@ -74,12 +74,15 @@ - - /* Define to 1 if you have the `ASN1_STRING_get0_data' function. */ - /* #undef HAVE_ASN1_STRING_GET0_DATA */ -+#define HAVE_ASN1_STRING_GET0_DATA 1 - - /* Define to 1 if you have the `BIO_get_data' function. */ - /* #undef HAVE_BIO_GET_DATA */ -+#define HAVE_BIO_GET_DATA 1 - - /* Define to 1 if you have the `BIO_meth_new' function. */ - /* #undef HAVE_BIO_METH_NEW */ -+#define HAVE_BIO_METH_NEW 1 - - /* Define to 1 if you have the `cbrt' function. */ - //#define HAVE_CBRT 1