psycopg/psycopg2
1
1
Fork 0
mirror of https://github.com/psycopg/psycopg2.git synced 2024-11-25 18:33:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/appveyor-py'

Browse Source
This commit is contained in:
Daniele Varrazzo 2019-05-01 16:30:09 +01:00
commit f96982bdfd
3 changed files with 921 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

219
.appveyor.yml
View File

@ -6,42 +6,44 @@ environment:
global:
# MSVC Express 2008's setenv.cmd failes if /E:ON and /V:ON are not
# enabled in the batch script interpreter
#
# WITH_COMPILER: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_compiler.cmd"
CMD_IN_ENV: cmd /E:ON /V:ON /C .\appveyor\run_with_env.cmd
matrix:
# For Python versions available on Appveyor, see
# https://www.appveyor.com/docs/build-environment/
- {PYVER: "27", PYTHON_ARCH: "32"}
- {PYVER: "27", PYTHON_ARCH: "64"}
- {PYVER: "37", PYTHON_ARCH: "32"}
- {PYVER: "37", PYTHON_ARCH: "64"}
- {PYVER: "36", PYTHON_ARCH: "32"}
- {PYVER: "36", PYTHON_ARCH: "64"}
- {PYVER: "35", PYTHON_ARCH: "32"}
- {PYVER: "35", PYTHON_ARCH: "64"}
- {PYVER: "34", PYTHON_ARCH: "32"}
- {PYVER: "34", PYTHON_ARCH: "64"}
- {PY_VER: "27", PY_ARCH: "32"}
- {PY_VER: "27", PY_ARCH: "64"}
- {PY_VER: "37", PY_ARCH: "32"}
- {PY_VER: "37", PY_ARCH: "64"}
- {PY_VER: "36", PY_ARCH: "32"}
- {PY_VER: "36", PY_ARCH: "64"}
- {PY_VER: "35", PY_ARCH: "32"}
- {PY_VER: "35", PY_ARCH: "64"}
- {PY_VER: "34", PY_ARCH: "32"}
- {PY_VER: "34", PY_ARCH: "64"}
OPENSSL_VERSION: "1_1_1b"
POSTGRES_VERSION: "11_2"
PSYCOPG2_TESTDB: psycopg2_test
PSYCOPG2_TESTDB_USER: postgres
PSYCOPG2_TESTDB_PASSWORD: Password12!
PSYCOPG2_TESTDB_HOST: localhost
PSYCOPG2_TESTDB_PORT: 5432
PGUSER: postgres
PGPASSWORD: Password12!
PGSSLMODE: require
# Select according to the service enabled
POSTGRES_DIR: C:\Program Files\PostgreSQL\9.6\
# The python used in the build process, not the one packages are built for
PYEXE: C:\Python36\python.exe
matrix:
fast_finish: false
services:
# Note: if you change this service also change the paths to match
# (see where Program Files\Postgres\9.6 is used)
# Note: if you change this service also change POSTGRES_DIR
- postgresql96
cache:
@ -50,189 +52,28 @@ cache:
- C:\Others -> scripts\appveyor.cache_rebuild
# Script called before repo cloning
init:
# Uncomment next line to get RDP access during the build.
#- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
# Set env variable according to the build environment
- SET PYTHON=C:\Python%PYVER%
- IF "%PYTHON_ARCH%"=="64" SET PYTHON=%PYTHON%-x64
# Py 2.7 = VS Ver. 9.0 (VS 2008)
# Py 3.4 = VS Ver. 10.0 (VS 2010)
# Py 3.5, 3.6, 3.7 = VS Ver. 14.0 (VS 2015)
- IF "%PYVER%"=="27" SET VS_VER=9.0
- IF "%PYVER%"=="34" SET VS_VER=10.0
- IF "%PYVER%"=="35" SET VS_VER=14.0
- IF "%PYVER%"=="36" SET VS_VER=14.0
- IF "%PYVER%"=="37" SET VS_VER=14.0
- IF "%VS_VER%"=="10.0" IF "%PYTHON_ARCH%"=="64" SET DISTUTILS_USE_SDK=1
# Set Python to the path
- SET PATH=%PYTHON%;%PYTHON%\Scripts;C:\Program Files\Git\mingw64\bin;%PATH%
# Verify Python version and architecture
- ECHO *******************************************************************
- ECHO Python Information
- ECHO *******************************************************************
- "%PYTHON%\\python --version"
- "%PYTHON%\\python -c \"import sys; print('64bit: ' + str(sys.maxsize > 2**32))\""
# Get & Install NASM
#- curl -L -o nasminst.exe https://www.nasm.us/pub/nasm/releasebuilds/2.12.02/win64/nasm-2.12.02-installer-x64.exe && start /wait nasminst.exe /S
#- SET PATH="C:\Program Files (x86)\nasm;%PATH%"
# Fix problem with VS2008 Express and 64bit builds
- ECHO Fixing VS2008 Express and 64bit builds
- COPY "C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\bin\\vcvars64.bat" "C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\bin\\amd64\\vcvarsamd64.bat"
# Fix problem with VS2010 Express 64bit missing vcvars64.bat
# Note: repository not cloned at this point, so need to fetch
# file another way
- ECHO Fixing VS2010 Express and 64bit builds
- curl -fsSL -o "C:\\Program Files (x86)\\Microsoft Visual Studio 10.0\\VC\\bin\\amd64\\vcvars64.bat" https://raw.githubusercontent.com/psycopg/psycopg2/master/scripts/vcvars64-vs2010.bat
# Setup the compiler based upon version and architecture
- ECHO Configuring Compiler
- IF "%PYTHON_ARCH%"=="32" (CALL "C:\\Program Files (x86)\\Microsoft Visual Studio %VS_VER%\\VC\\vcvarsall.bat" x86)
- IF "%PYTHON_ARCH%"=="64" (CALL "C:\\Program Files (x86)\\Microsoft Visual Studio %VS_VER%\\VC\\vcvarsall.bat" amd64)
# The program rc.exe on 64bit with some versions look in the wrong path
# location when building postgresql. This cheats by copying the x64 bit
# files to that location.
- IF "%PYTHON_ARCH%"=="64" (COPY /Y "C:\\Program Files\\Microsoft SDKs\\Windows\\v7.0\\Bin\\x64\\rc*" "C:\\Program Files (x86)\\Microsoft SDKs\\Windows\\v7.0A\\Bin")
# Change PostgreSQL config before service starts to allow > 1 prepared
# transactions for test cases
- ECHO max_prepared_transactions = 10 >> "C:\\Program Files\\PostgreSQL\\9.6\\data\\postgresql.conf"
# init:
# Repository gets cloned, Cache is restored
install:
# We start off CD'ed to cloned folder
- SET BASE_DIR=C:\Others\%PYTHON_ARCH%\%VS_VER%
- SET BUILD_DIR=%BASE_DIR%\Builds
- IF NOT EXIST %BUILD_DIR% MKDIR %BUILD_DIR%
- "%PYEXE% scripts\\appveyor.py install"
- ECHO *******************************************************************
- ECHO Initialized variables specific for this build
- ECHO *******************************************************************
- ECHO %BASE_DIR%
- ECHO %BUILD_DIR%
- ECHO *******************************************************************
# Setup directories for building OpenSSL libraries
- ECHO *******************************************************************
- ECHO Preparing for building OpenSSL
- ECHO *******************************************************************
- SET OPENSSLTOP=%BASE_DIR%\openssl
- IF NOT EXIST %OPENSSLTOP%\include\openssl MKDIR %OPENSSLTOP%\include\openssl
- IF NOT EXIST %OPENSSLTOP%\lib MKDIR %OPENSSLTOP%\lib
# Setup OpenSSL Environment Variables based on processor architecture
- ps: >-
If ($env:PYTHON_ARCH -Match "32" ) {
$env:VCVARS_PLATFORM="x86"
$env:TARGET="VC-WIN32"
} Else {
$env:VCVARS_PLATFORM="amd64"
$env:TARGET="VC-WIN64A"
$env:CPU="AMD64"
}
# Download OpenSSL source
- CD C:\Others
- IF NOT EXIST OpenSSL_%OPENSSL_VERSION%.zip (
curl -fsSL -o OpenSSL_%OPENSSL_VERSION%.zip https://github.com/openssl/openssl/archive/OpenSSL_%OPENSSL_VERSION%.zip
)
- IF NOT EXIST %OPENSSLTOP%\lib\libssl.lib (
CD %BUILD_DIR% &&
7z x C:\Others\OpenSSL_%OPENSSL_VERSION%.zip &&
CD openssl-OpenSSL_%OPENSSL_VERSION% &&
perl Configure %TARGET% no-asm no-shared no-zlib --prefix=%OPENSSLTOP% --openssldir=%OPENSSLTOP% &&
nmake build_libs install_dev &&
CD %BASE_DIR% &&
RMDIR /S /Q %BUILD_DIR%\openssl-OpenSSL_%OPENSSL_VERSION%
)
# Setup directories for building PostgreSQL librarires
- ECHO *******************************************************************
- ECHO Preparing for building PostgreSQL libraries
- ECHO *******************************************************************
- SET PGTOP=%BASE_DIR%\postgresql
- IF NOT EXIST %PGTOP%\include MKDIR %PGTOP%\include
- IF NOT EXIST %PGTOP%\lib MKDIR %PGTOP%\lib
- IF NOT EXIST %PGTOP%\bin MKDIR %PGTOP%\bin
# Download PostgreSQL source
- CD C:\Others
- IF NOT EXIST postgres-REL_%POSTGRES_VERSION%.zip (
curl -fsSL -o postgres-REL_%POSTGRES_VERSION%.zip https://github.com/postgres/postgres/archive/REL_%POSTGRES_VERSION%.zip
)
# Setup build config file (config.pl)
# Hack the Mkvcbuild.pm file so we build the lib version of libpq
# Build libpgport, libpgcommon, libpq
# Install includes
# Copy over built libraries
# Prepare local include directory for building from
# Build pg_config in place
# Note patch for OpenSSL 1.1 configuration. See:
# https://www.postgresql-archive.org/Compile-psql-9-6-with-SSL-Version-1-1-0-td6054118.html
# NOTE: Cannot set and use the same variable inside an IF
- SET PGBUILD=%BUILD_DIR%\postgres-REL_%POSTGRES_VERSION%
- IF NOT EXIST %PGTOP%\lib\libpq.lib (
CD %BUILD_DIR% &&
7z x C:\Others\postgres-REL_%POSTGRES_VERSION%.zip &&
CD postgres-REL_%POSTGRES_VERSION% &&
patch -p1 < %APPVEYOR_BUILD_FOLDER%\scripts\win_openssl_11.patch &&
CD src\tools\msvc &&
ECHO $config-^>{ldap} = 0; > config.pl &&
ECHO $config-^>{openssl} = "%OPENSSLTOP:\=\\%"; >> config.pl &&
ECHO.>> config.pl &&
ECHO 1;>> config.pl &&
perl -pi.bak -e "s/'libpq', 'dll'/'libpq', 'lib'/g" Mkvcbuild.pm &&
build libpgport &&
build libpgcommon &&
build libpq &&
ECHO "" > %PGBUILD%\src\backend\parser\gram.h &&
perl -pi.bak -e "s/qw\(Install\)/qw\(Install CopyIncludeFiles\)/g" Install.pm &&
perl -MInstall=CopyIncludeFiles -e"chdir('../../..'); CopyIncludeFiles('%PGTOP%')" &&
COPY %PGBUILD%\Release\libpgport\libpgport.lib %PGTOP%\lib &&
COPY %PGBUILD%\Release\libpgcommon\libpgcommon.lib %PGTOP%\lib &&
COPY %PGBUILD%\Release\libpq\libpq.lib %PGTOP%\lib &&
XCOPY /Y /S %PGBUILD%\src\include\port\win32\* %PGBUILD%\src\include &&
XCOPY /Y /S %PGBUILD%\src\include\port\win32_msvc\* %PGBUILD%\src\include &&
CD %PGBUILD%\src\bin\pg_config &&
cl pg_config.c /MT /nologo /I%PGBUILD%\src\include /link /LIBPATH:%PGTOP%\lib libpgcommon.lib libpgport.lib advapi32.lib /NODEFAULTLIB:libcmt.lib /OUT:%PGTOP%\bin\pg_config.exe &&
CD %BASE_DIR% &&
RMDIR /S /Q %PGBUILD%
)
# PostgreSQL server starts now
build: off
#before_build:
build_script:
# Add PostgreSQL binaries to the path
- PATH=C:\Program Files\PostgreSQL\9.6\bin\;%PATH%
- CD C:\Project
- "%PYTHON%\\python.exe setup.py build_ext --have-ssl --pg-config %PGTOP%\\bin\\pg_config.exe -l libpgcommon -l libpgport -L %OPENSSLTOP%\\lib -I %OPENSSLTOP%\\include"
- "%PYTHON%\\python.exe setup.py build"
- "%PYTHON%\\python.exe setup.py install"
- RD /S /Q psycopg2.egg-info
- "%PYEXE% scripts\\appveyor.py build_script"
#after_build:
after_build:
- "%PYEXE% scripts\\appveyor.py after_build"
before_test:
# Create and setup PostgreSQL database for the tests
- createdb %PSYCOPG2_TESTDB%
- psql -d %PSYCOPG2_TESTDB% -c "CREATE EXTENSION HSTORE;"
- "%PYEXE% scripts\\appveyor.py before_test"
test_script:
# Print psycopg and libpq versions
- "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.__version__)\""
- "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.__libpq_version__)\""
- "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.extensions.libpq_version())\""
- "%PYTHON%\\python.exe -c \"import tests; tests.unittest.main(defaultTest='tests.test_suite')\" --verbose"
- "%PYEXE% scripts\\appveyor.py test_script"
# vim: set ts=4 sts=4 sw=4:

891
scripts/appveyor.py Executable file
View File

@ -0,0 +1,891 @@
#!/usr/bin/env python3
"""
Build steps for the windows binary packages.
The script is designed to be called by appveyor. Subcommands map the steps in
'appveyor.yml'.
"""
import re
import os
import sys
import json
import shutil
import logging
import subprocess as sp
from glob import glob
from pathlib import Path
from zipfile import ZipFile
from tempfile import NamedTemporaryFile
from urllib.request import urlopen
opt = None
STEP_PREFIX = 'step_'
logger = logging.getLogger()
logging.basicConfig(
level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s'
)
def main():
global opt
opt = parse_cmdline()
logger.setLevel(opt.loglevel)
cmd = globals()[STEP_PREFIX + opt.step]
cmd()
def setup_build_env():
"""
Set the environment variables according to the build environment
"""
setenv('VS_VER', opt.vs_ver)
if opt.vs_ver == '10.0' and opt.arch_64:
setenv('DISTUTILS_USE_SDK', '1')
path = [
str(opt.py_dir),
str(opt.py_dir / 'Scripts'),
r'C:\Program Files\Git\mingw64\bin',
os.environ['PATH'],
]
setenv('PATH', os.pathsep.join(path))
if opt.vs_ver == '9.0':
logger.info("Fixing VS2008 Express and 64bit builds")
shutil.copyfile(
opt.vc_dir / "bin/vcvars64.bat",
opt.vc_dir / "bin/amd64/vcvarsamd64.bat",
)
# Fix problem with VS2010 Express 64bit missing vcvars64.bat
if opt.vs_ver == '10.0':
if not (opt.vc_dir / "bin/amd64/vcvars64.bat").exists():
logger.info("Fixing VS2010 Express and 64bit builds")
copy_file(
opt.package_dir / "scripts/vcvars64-vs2010.bat",
opt.vc_dir / "bin/amd64/vcvars64.bat",
)
logger.info("Configuring compiler")
bat_call([opt.vc_dir / "vcvarsall.bat", 'x86' if opt.arch_32 else 'amd64'])
def python_info():
logger.info("Python Information")
run_python(['--version'], stderr=sp.STDOUT)
run_python(
['-c', "import sys; print('64bit: %s' % (sys.maxsize > 2**32))"]
)
def step_install():
python_info()
configure_sdk()
configure_postgres()
if opt.is_wheel:
install_wheel_support()
def install_wheel_support():
"""
Install an up-to-date pip wheel package to build wheels.
"""
run_python("-m pip install --upgrade pip".split())
run_python("-m pip install wheel".split())
def configure_sdk():
# The program rc.exe on 64bit with some versions look in the wrong path
# location when building postgresql. This cheats by copying the x64 bit
# files to that location.
if opt.arch_64:
for fn in glob(
r'C:\Program Files\Microsoft SDKs\Windows\v7.0\Bin\x64\rc*'
):
copy_file(
fn, r"C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Bin"
)
def configure_postgres():
"""
Set up PostgreSQL config before the service starts.
"""
logger.info("Configuring Postgres")
with (opt.pg_data_dir / 'postgresql.conf').open('a') as f:
# allow > 1 prepared transactions for test cases
print("max_prepared_transactions = 10", file=f)
print("ssl = on", file=f)
# Create openssl certificate to allow ssl connection
cwd = os.getcwd()
os.chdir(opt.pg_data_dir)
run_openssl(
'req -new -x509 -days 365 -nodes -text '
'-out server.crt -keyout server.key -subj /CN=initd.org'.split()
)
run_openssl(
'req -new -nodes -text -out root.csr -keyout root.key '
'-subj /CN=initd.org'.split()
)
run_openssl(
'x509 -req -in root.csr -text -days 3650 -extensions v3_ca '
'-signkey root.key -out root.crt'.split()
)
run_openssl(
'req -new -nodes -text -out server.csr -keyout server.key '
'-subj /CN=initd.org'.split()
)
run_openssl(
'x509 -req -in server.csr -text -days 365 -CA root.crt '
'-CAkey root.key -CAcreateserial -out server.crt'.split()
)
os.chdir(cwd)
def run_openssl(args):
"""Run the appveyor-installed openssl with some args."""
# https://www.appveyor.com/docs/windows-images-software/
openssl = Path(r"C:\OpenSSL-v111-Win64") / 'bin' / 'openssl'
return run_command([openssl] + args)
def step_build_script():
setup_build_env()
build_openssl()
build_libpq()
build_psycopg()
if opt.is_wheel:
build_binary_packages()
def build_openssl():
top = opt.ssl_build_dir
if (top / 'lib' / 'libssl.lib').exists():
return
logger.info("Building OpenSSL")
# Setup directories for building OpenSSL libraries
ensure_dir(top / 'include' / 'openssl')
ensure_dir(top / 'lib')
# Setup OpenSSL Environment Variables based on processor architecture
if opt.arch_32:
target = 'VC-WIN32'
setenv('VCVARS_PLATFORM', 'x86')
else:
target = 'VC-WIN64A'
setenv('VCVARS_PLATFORM', 'amd64')
setenv('CPU', 'AMD64')
ver = os.environ['OPENSSL_VERSION']
# Download OpenSSL source
zipname = f'OpenSSL_{ver}.zip'
zipfile = opt.cache_dir / zipname
if not zipfile.exists():
download(
f"https://github.com/openssl/openssl/archive/{zipname}", zipfile
)
with ZipFile(zipfile) as z:
z.extractall(path=opt.build_dir)
sslbuild = opt.build_dir / f"openssl-OpenSSL_{ver}"
os.chdir(sslbuild)
run_command(
['perl', 'Configure', target, 'no-asm']
+ ['no-shared', 'no-zlib', f'--prefix={top}', f'--openssldir={top}']
)
run_command("nmake build_libs install_dev".split())
assert (top / 'lib' / 'libssl.lib').exists()
os.chdir(opt.clone_dir)
shutil.rmtree(sslbuild)
def build_libpq():
top = opt.pg_build_dir
if (top / 'lib' / 'libpq.lib').exists():
return
logger.info("Building libpq")
# Setup directories for building PostgreSQL librarires
ensure_dir(top / 'include')
ensure_dir(top / 'lib')
ensure_dir(top / 'bin')
ver = os.environ['POSTGRES_VERSION']
# Download PostgreSQL source
zipname = f'postgres-REL_{ver}.zip'
zipfile = opt.cache_dir / zipname
if not zipfile.exists():
download(
f"https://github.com/postgres/postgres/archive/REL_{ver}.zip",
zipfile,
)
with ZipFile(zipfile) as z:
z.extractall(path=opt.build_dir)
pgbuild = opt.build_dir / f"postgres-REL_{ver}"
os.chdir(pgbuild)
# Patch for OpenSSL 1.1 configuration. See:
# https://www.postgresql-archive.org/Compile-psql-9-6-with-SSL-Version-1-1-0-td6054118.html
assert Path("src/include/pg_config.h.win32").exists()
with open("src/include/pg_config.h.win32", 'a') as f:
print(
"""
#define HAVE_ASN1_STRING_GET0_DATA 1
#define HAVE_BIO_GET_DATA 1
#define HAVE_BIO_METH_NEW 1
#define HAVE_OPENSSL_INIT_SSL 1
""",
file=f,
)
# Setup build config file (config.pl)
os.chdir("src/tools/msvc")
with open("config.pl", 'w') as f:
print(
"""\
$config->{ldap} = 0;
$config->{openssl} = "%s";
1;
"""
% str(opt.ssl_build_dir).replace('\\', '\\\\'),
file=f,
)
# Hack the Mkvcbuild.pm file so we build the lib version of libpq
file_replace('Mkvcbuild.pm', "'libpq', 'dll'", "'libpq', 'lib'")
# Build libpgport, libpgcommon, libpq
run_command([which("build"), "libpgport"])
run_command([which("build"), "libpgcommon"])
run_command([which("build"), "libpq"])
# Install includes
with (pgbuild / "src/backend/parser/gram.h").open("w") as f:
print("", file=f)
# Copy over built libraries
file_replace("Install.pm", "qw(Install)", "qw(Install CopyIncludeFiles)")
run_command(
["perl", "-MInstall=CopyIncludeFiles", "-e"]
+ [f"chdir('../../..'); CopyIncludeFiles('{top}')"]
)
for lib in ('libpgport', 'libpgcommon', 'libpq'):
copy_file(pgbuild / f'Release/{lib}/{lib}.lib', top / 'lib')
# Prepare local include directory for building from
for dir in ('win32', 'win32_msvc'):
merge_dir(pgbuild / f"src/include/port/{dir}", pgbuild / "src/include")
# Build pg_config in place
os.chdir(pgbuild / 'src/bin/pg_config')
run_command(
['cl', 'pg_config.c', '/MT', '/nologo', fr'/I{pgbuild}\src\include']
+ ['/link', fr'/LIBPATH:{top}\lib']
+ ['libpgcommon.lib', 'libpgport.lib', 'advapi32.lib']
+ ['/NODEFAULTLIB:libcmt.lib']
+ [fr'/OUT:{top}\bin\pg_config.exe']
)
assert (top / 'lib' / 'libpq.lib').exists()
assert (top / 'bin' / 'pg_config.exe').exists()
os.chdir(opt.clone_dir)
shutil.rmtree(pgbuild)
def build_psycopg():
os.chdir(opt.package_dir)
patch_package_name()
add_pg_config_path()
run_python(
["setup.py", "build_ext", "--have-ssl"]
+ ["-l", "libpgcommon", "-l", "libpgport"]
+ ["-L", opt.ssl_build_dir / 'lib']
+ ['-I', opt.ssl_build_dir / 'include']
)
run_python(["setup.py", "build_py"])
def patch_package_name():
"""Change the psycopg2 package name in the setup.py if required."""
if opt.package_name == 'psycopg2':
return
logger.info("changing package name to %s", opt.package_name)
with (opt.package_dir / 'setup.py').open() as f:
data = f.read()
# Replace the name of the package with what desired
rex = re.compile(r"""name=["']psycopg2["']""")
assert len(rex.findall(data)) == 1, rex.findall(data)
data = rex.sub(f'name="{opt.package_name}"', data)
with (opt.package_dir / 'setup.py').open('w') as f:
f.write(data)
def build_binary_packages():
"""Create wheel/exe binary packages."""
os.chdir(opt.package_dir)
add_pg_config_path()
# Build .exe packages for whom still use them
if opt.package_name == 'psycopg2':
run_python(['setup.py', 'bdist_wininst', "-d", opt.dist_dir])
# Build .whl packages
run_python(['setup.py', 'bdist_wheel', "-d", opt.dist_dir])
def step_after_build():
if not opt.is_wheel:
install_built_package()
else:
install_binary_package()
def install_built_package():
"""Install the package just built by setup build."""
os.chdir(opt.package_dir)
# Install the psycopg just built
add_pg_config_path()
run_python(["setup.py", "install"])
shutil.rmtree("psycopg2.egg-info")
def install_binary_package():
"""Install the package from a packaged wheel."""
run_python(
['-m', 'pip', 'install', '--no-index', '-f', opt.dist_dir]
+ [opt.package_name]
)
def add_pg_config_path():
"""Allow finding in the path the pg_config just built."""
pg_path = str(opt.pg_build_dir / 'bin')
if pg_path not in os.environ['PATH'].split(os.pathsep):
setenv('PATH', os.pathsep.join([pg_path, os.environ['PATH']]))
def step_before_test():
print_psycopg2_version()
# Create and setup PostgreSQL database for the tests
run_command([opt.pg_bin_dir / 'createdb', os.environ['PSYCOPG2_TESTDB']])
run_command(
[opt.pg_bin_dir / 'psql', '-d', os.environ['PSYCOPG2_TESTDB']]
+ ['-c', "CREATE EXTENSION hstore"]
)
def print_psycopg2_version():
"""Print psycopg2 and libpq versions installed."""
for expr in (
'psycopg2.__version__',
'psycopg2.__libpq_version__',
'psycopg2.extensions.libpq_version()',
):
out = out_python(['-c', f"import psycopg2; print({expr})"])
logger.info("built %s: %s", expr, out.decode('ascii'))
def step_test_script():
check_libpq_version()
run_test_suite()
def check_libpq_version():
"""
Fail if the package installed is not using the expected libpq version.
"""
want_ver = tuple(map(int, os.environ['POSTGRES_VERSION'].split('_')))
want_ver = "%d%04d" % want_ver
got_ver = (
out_python(
['-c']
+ ["import psycopg2; print(psycopg2.extensions.libpq_version())"]
)
.decode('ascii')
.rstrip()
)
assert want_ver == got_ver, "libpq version mismatch: %r != %r" % (
want_ver,
got_ver,
)
def run_test_suite():
# Remove this var, which would make badly a configured OpenSSL 1.1 work
os.environ.pop('OPENSSL_CONF', None)
# Run the unit test
args = [
'-c',
"import tests; tests.unittest.main(defaultTest='tests.test_suite')",
]
if opt.is_wheel:
os.environ['PSYCOPG2_TEST_FAST'] = '1'
else:
args.append('--verbose')
os.chdir(opt.package_dir)
run_python(args)
def step_on_success():
print_sha1_hashes()
if setup_ssh():
upload_packages()
def print_sha1_hashes():
"""
Print the packages sha1 so their integrity can be checked upon signing.
"""
logger.info("artifacts SHA1 hashes:")
os.chdir(opt.package_dir / 'dist')
run_command([which('sha1sum'), '-b', f'psycopg2-*/*'])
def setup_ssh():
"""
Configure ssh to upload built packages where they can be retrieved.
Return False if can't configure and upload shoould be skipped.
"""
# If we are not on the psycopg AppVeyor account, the environment variable
# REMOTE_KEY will not be decrypted. In that case skip uploading.
if os.environ['APPVEYOR_ACCOUNT_NAME'] != 'psycopg':
logger.warn("skipping artifact upload: you are not psycopg")
return False
pkey = os.environ.get('REMOTE_KEY', None)
if not pkey:
logger.warn("skipping artifact upload: no remote key")
return False
# Write SSH Private Key file from environment variable
pkey = pkey.replace(' ', '\n')
with (opt.clone_dir / 'id_rsa').open('w') as f:
f.write(
f"""\
-----BEGIN RSA PRIVATE KEY-----
{pkey}
-----END RSA PRIVATE KEY-----
"""
)
# Make a directory to please MinGW's version of ssh
ensure_dir(r"C:\MinGW\msys\1.0\home\appveyor\.ssh")
return True
def upload_packages():
# Upload built artifacts
logger.info("uploading artifacts")
ssh_cmd = r"C:\MinGW\msys\1.0\bin\ssh -i %s -o UserKnownHostsFile=%s" % (
opt.clone_dir / "id_rsa",
opt.clone_dir / 'known_hosts',
)
os.chdir(opt.package_dir)
run_command(
[r"C:\MinGW\msys\1.0\bin\rsync", "-avr"]
+ ["-e", ssh_cmd, "dist/", "upload@initd.org:"]
)
def download(url, fn):
"""Download a file locally"""
logger.info("downloading %s", url)
with open(fn, 'wb') as fo, urlopen(url) as fi:
while 1:
data = fi.read(8192)
if not data:
break
fo.write(data)
logger.info("file downloaded: %s", fn)
def file_replace(fn, s1, s2):
"""
Replace all the occurrences of the string s1 into s2 in the file fn.
"""
assert os.path.exists(fn)
with open(fn, 'r+') as f:
data = f.read()
f.seek(0)
f.write(data.replace(s1, s2))
f.truncate()
def merge_dir(src, tgt):
"""
Merge the content of the directory src into the directory tgt
Reproduce the semantic of "XCOPY /Y /S src/* tgt"
"""
src = str(src)
for dp, _dns, fns in os.walk(src):
logger.debug("dirpath %s", dp)
if not fns:
continue
assert dp.startswith(src)
subdir = dp[len(src) :].lstrip(os.sep)
tgtdir = ensure_dir(os.path.join(tgt, subdir))
for fn in fns:
copy_file(os.path.join(dp, fn), tgtdir)
def bat_call(cmdline):
"""
Simulate 'CALL' from a batch file
Execute CALL *cmdline* and export the changed environment to the current
environment.
nana-nana-nana-nana...
"""
if not isinstance(cmdline, str):
cmdline = map(str, cmdline)
cmdline = ' '.join(c if ' ' not in c else '"%s"' % c for c in cmdline)
data = f"""\
CALL {cmdline}
{opt.py_exe} -c "import os, sys, json; \
json.dump(dict(os.environ), sys.stdout, indent=2)"
"""
logger.debug("preparing file to batcall:\n\n%s", data)
with NamedTemporaryFile(suffix='.bat') as tmp:
fn = tmp.name
with open(fn, "w") as f:
f.write(data)
try:
out = out_command(fn)
# be vewwy vewwy caweful to print the env var as it might contain
# secwet things like your pwecious pwivate key.
# logger.debug("output of command:\n\n%s", out.decode('utf8', 'replace'))
# The output has some useless crap on stdout, because sure, and json
# indented so the last { on column 1 is where we have to start parsing
m = list(re.finditer(b'^{', out, re.MULTILINE))[-1]
out = out[m.start() :]
env = json.loads(out)
for k, v in env.items():
if os.environ.get(k) != v:
setenv(k, v)
finally:
os.remove(fn)
def ensure_dir(dir):
if not isinstance(dir, Path):
dir = Path(dir)
if not dir.is_dir():
logger.info("creating directory %s", dir)
dir.mkdir(parents=True)
return dir
def run_command(cmdline, **kwargs):
"""Run a command, raise on error."""
if not isinstance(cmdline, str):
cmdline = list(map(str, cmdline))
logger.info("running command: %s", cmdline)
sp.check_call(cmdline, **kwargs)
def out_command(cmdline, **kwargs):
"""Run a command, return its output, raise on error."""
if not isinstance(cmdline, str):
cmdline = list(map(str, cmdline))
logger.info("running command: %s", cmdline)
data = sp.check_output(cmdline, **kwargs)
return data
def run_python(args, **kwargs):
"""
Run a script in the target Python.
"""
return run_command([opt.py_exe] + args, **kwargs)
def out_python(args, **kwargs):
"""
Return the output of a script run in the target Python.
"""
return out_command([opt.py_exe] + args, **kwargs)
def copy_file(src, dst):
logger.info("copying file %s -> %s", src, dst)
shutil.copy(src, dst)
def setenv(k, v):
logger.debug("setting %s=%s", k, v)
os.environ[k] = v
def which(name):
"""
Return the full path of a command found on the path
"""
base, ext = os.path.splitext(name)
if not ext:
exts = ('.com', '.exe', '.bat', '.cmd')
else:
exts = (ext,)
for dir in ['.'] + os.environ['PATH'].split(os.pathsep):
for ext in exts:
fn = os.path.join(dir, base + ext)
if os.path.isfile(fn):
return fn
raise Exception("couldn't find program on path: %s" % name)
class Options:
"""
An object exposing the script configuration from env vars and command line.
"""
@property
def py_ver(self):
"""The Python version to build as 2 digits string."""
rv = os.environ['PY_VER']
assert rv in ('27', '34', '35', '36', '37'), rv
return rv
@property
def py_arch(self):
"""The Python architecture to build, 32 or 64."""
rv = os.environ['PY_ARCH']
assert rv in ('32', '64'), rv
return int(rv)
@property
def arch_32(self):
"""True if the Python architecture to build is 32 bits."""
return self.py_arch == 32
@property
def arch_64(self):
"""True if the Python architecture to build is 64 bits."""
return self.py_arch == 64
@property
def package_name(self):
return os.environ.get('CONFIGURATION', 'psycopg2')
@property
def package_version(self):
"""The psycopg2 version number to build."""
with (self.package_dir / 'setup.py').open() as f:
data = f.read()
m = re.search(
r"""^PSYCOPG_VERSION\s*=\s*['"](.*)['"]""", data, re.MULTILINE
)
return m.group(1)
@property
def is_wheel(self):
"""Are we building the wheel packages or just the extension?"""
project_name = os.environ['APPVEYOR_PROJECT_NAME']
if project_name == 'psycopg2':
return False
elif project_name == 'psycopg2-wheels':
return True
else:
raise Exception(f"unexpected project name: {project_name}")
@property
def py_dir(self):
"""
The path to the target python binary to execute.
"""
dirname = ''.join(
[r"C:\Python", self.py_ver, '-x64' if self.arch_64 else '']
)
return Path(dirname)
@property
def py_exe(self):
"""
The full path of the target python executable.
"""
return self.py_dir / 'python.exe'
@property
def vc_dir(self):
"""
The path of the Visual C compiler.
"""
return Path(
r"C:\Program Files (x86)\Microsoft Visual Studio %s\VC"
% self.vs_ver
)
@property
def vs_ver(self):
# Py 2.7 = VS Ver. 9.0 (VS 2008)
# Py 3.3, 3.4 = VS Ver. 10.0 (VS 2010)
# Py 3.5, 3.6, 3.7 = VS Ver. 14.0 (VS 2015)
vsvers = {
'27': '9.0',
'33': '10.0',
'34': '10.0',
'35': '14.0',
'36': '14.0',
'37': '14.0',
}
return vsvers[self.py_ver]
@property
def clone_dir(self):
"""The directory where the repository is cloned."""
return Path(r"C:\Project")
@property
def appveyor_pg_dir(self):
"""The directory of the postgres service made available by Appveyor."""
return Path(os.environ['POSTGRES_DIR'])
@property
def pg_data_dir(self):
"""The data dir of the appveyor postgres service."""
return self.appveyor_pg_dir / 'data'
@property
def pg_bin_dir(self):
"""The bin dir of the appveyor postgres service."""
return self.appveyor_pg_dir / 'bin'
@property
def pg_build_dir(self):
"""The directory where to build the postgres libraries for psycopg."""
return self.cache_arch_dir / 'postgresql'
@property
def ssl_build_dir(self):
"""The directory where to build the openssl libraries for psycopg."""
return self.cache_arch_dir / 'openssl'
@property
def cache_arch_dir(self):
rv = self.cache_dir / str(self.py_arch) / self.vs_ver
return ensure_dir(rv)
@property
def cache_dir(self):
return Path(r"C:\Others")
@property
def build_dir(self):
rv = self.cache_arch_dir / 'Builds'
return ensure_dir(rv)
@property
def package_dir(self):
"""
The directory containing the psycopg code checkout dir.
Building psycopg it is clone_dir, building the wheels it is a submodule.
"""
return self.clone_dir / 'psycopg2' if self.is_wheel else self.clone_dir
@property
def dist_dir(self):
"""The directory where to build packages to distribute."""
return (
self.package_dir / 'dist' / ('psycopg2-%s' % self.package_version)
)
def parse_cmdline():
from argparse import ArgumentParser
parser = ArgumentParser(description=__doc__)
g = parser.add_mutually_exclusive_group()
g.add_argument(
'-q',
'--quiet',
help="Talk less",
dest='loglevel',
action='store_const',
const=logging.WARN,
default=logging.INFO,
)
g.add_argument(
'-v',
'--verbose',
help="Talk more",
dest='loglevel',
action='store_const',
const=logging.DEBUG,
default=logging.INFO,
)
steps = [
n[len(STEP_PREFIX) :]
for n in globals()
if n.startswith(STEP_PREFIX) and callable(globals()[n])
]
parser.add_argument(
'step', choices=steps, help="the appveyor step to execute"
)
opt = parser.parse_args(namespace=Options())
return opt
if __name__ == '__main__':
sys.exit(main())

20
scripts/win_openssl_11.patch
View File

@ -1,20 +0,0 @@
diff --git a/src/include/pg_config.h.win32 b/src/include/pg_config.h.win32
index dfd6972383..72d1259ffb 100644
--- a/src/include/pg_config.h.win32
+++ b/src/include/pg_config.h.win32
@@ -74,12 +74,15 @@
/* Define to 1 if you have the `ASN1_STRING_get0_data' function. */
/* #undef HAVE_ASN1_STRING_GET0_DATA */
+#define HAVE_ASN1_STRING_GET0_DATA 1
/* Define to 1 if you have the `BIO_get_data' function. */
/* #undef HAVE_BIO_GET_DATA */
+#define HAVE_BIO_GET_DATA 1
/* Define to 1 if you have the `BIO_meth_new' function. */
/* #undef HAVE_BIO_METH_NEW */
+#define HAVE_BIO_METH_NEW 1
/* Define to 1 if you have the `cbrt' function. */
//#define HAVE_CBRT 1