psycopg/psycopg2
1
1
Fork 0
mirror of https://github.com/psycopg/psycopg2.git synced 2024-11-22 17:06:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added script to implement appveyor functionality in Python

Browse Source 
Only the init step for the moment.
This commit is contained in:
Daniele Varrazzo 2019-04-14 20:10:51 +01:00
parent 325aadbf2c
commit 00fc2820a0
2 changed files with 289 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
.appveyor.yml
View File

@ -6,8 +6,6 @@ environment:
global:
# MSVC Express 2008's setenv.cmd failes if /E:ON and /V:ON are not
# enabled in the batch script interpreter
#
# WITH_COMPILER: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_compiler.cmd"
CMD_IN_ENV: cmd /E:ON /V:ON /C .\appveyor\run_with_env.cmd
matrix:
@ -29,13 +27,14 @@ environment:
PSYCOPG2_TESTDB: psycopg2_test
PSYCOPG2_TESTDB_USER: postgres
PSYCOPG2_TESTDB_PASSWORD: Password12!
PSYCOPG2_TESTDB_HOST: localhost
PSYCOPG2_TESTDB_PORT: 5432
PGUSER: postgres
PGPASSWORD: Password12!
# The python used in the build process, not the one packages are built for
PYEXE: C:\Python36\python.exe
matrix:
fast_finish: false
@ -51,8 +50,12 @@ cache:
# Script called before repo cloning
init:
# Uncomment next line to get RDP access during the build.
#- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
# TODO: move functionalities out of init to drop this fetch problem
- curl -fsSL -o "C:\\appveyor.py" https://raw.githubusercontent.com/psycopg/psycopg2/%APPVEYOR_REPO_COMMIT%/scripts/appveyor.py
- "%PYEXE% C:\\appveyor.py init"
# TODO: exporting variables below to be dropped once all the steps are
# moved to the appveyor.py script.
# Set env variable according to the build environment
- SET PYTHON=C:\Python%PYVER%
@ -72,41 +75,11 @@ init:
# Set Python to the path
- SET PATH=%PYTHON%;%PYTHON%\Scripts;C:\Program Files\Git\mingw64\bin;%PATH%
# Verify Python version and architecture
- ECHO *******************************************************************
- ECHO Python Information
- ECHO *******************************************************************
- "%PYTHON%\\python --version"
- "%PYTHON%\\python -c \"import sys; print('64bit: ' + str(sys.maxsize > 2**32))\""
# Get & Install NASM
#- curl -L -o nasminst.exe https://www.nasm.us/pub/nasm/releasebuilds/2.12.02/win64/nasm-2.12.02-installer-x64.exe && start /wait nasminst.exe /S
#- SET PATH="C:\Program Files (x86)\nasm;%PATH%"
# Fix problem with VS2008 Express and 64bit builds
- ECHO Fixing VS2008 Express and 64bit builds
- COPY "C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\bin\\vcvars64.bat" "C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\VC\\bin\\amd64\\vcvarsamd64.bat"
# Fix problem with VS2010 Express 64bit missing vcvars64.bat
# Note: repository not cloned at this point, so need to fetch
# file another way
- ECHO Fixing VS2010 Express and 64bit builds
- curl -fsSL -o "C:\\Program Files (x86)\\Microsoft Visual Studio 10.0\\VC\\bin\\amd64\\vcvars64.bat" https://raw.githubusercontent.com/psycopg/psycopg2/master/scripts/vcvars64-vs2010.bat
# Setup the compiler based upon version and architecture
- ECHO Configuring Compiler
- IF "%PYTHON_ARCH%"=="32" (CALL "C:\\Program Files (x86)\\Microsoft Visual Studio %VS_VER%\\VC\\vcvarsall.bat" x86)
- IF "%PYTHON_ARCH%"=="64" (CALL "C:\\Program Files (x86)\\Microsoft Visual Studio %VS_VER%\\VC\\vcvarsall.bat" amd64)
# The program rc.exe on 64bit with some versions look in the wrong path
# location when building postgresql. This cheats by copying the x64 bit
# files to that location.
- IF "%PYTHON_ARCH%"=="64" (COPY /Y "C:\\Program Files\\Microsoft SDKs\\Windows\\v7.0\\Bin\\x64\\rc*" "C:\\Program Files (x86)\\Microsoft SDKs\\Windows\\v7.0A\\Bin")
# Change PostgreSQL config before service starts to allow > 1 prepared
# transactions for test cases
- ECHO max_prepared_transactions = 10 >> "C:\\Program Files\\PostgreSQL\\9.6\\data\\postgresql.conf"
# Repository gets cloned, Cache is restored
install:
@ -236,3 +209,6 @@ test_script:
- "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.__libpq_version__)\""
- "%PYTHON%\\python.exe -c \"import psycopg2; print(psycopg2.extensions.libpq_version())\""
- "%PYTHON%\\python.exe -c \"import tests; tests.unittest.main(defaultTest='tests.test_suite')\" --verbose"
# vim: set ts=4 sts=4 sw=4:

277
scripts/appveyor.py Executable file
View File

@ -0,0 +1,277 @@
#!/usr/bin/env python3
"""
Build steps for the windows binary packages.
The script is designed to be called by appveyor. Subcommands map the steps in
'appveyor.yml'.
"""
import re
import os
import sys
import json
import shutil
import logging
import subprocess as sp
from glob import glob
from urllib.request import urlopen
from tempfile import NamedTemporaryFile
from functools import lru_cache
opt = None
STEP_PREFIX = 'step_'
logger = logging.getLogger()
logging.basicConfig(
level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s'
)
def main():
global opt
opt = parse_cmdline()
setup_env()
cmd = globals()[STEP_PREFIX + opt.step]
cmd()
@lru_cache()
def setup_env():
"""
Set the environment variables according to the build environment
"""
python_info()
os.environ['VS_VER'] = vs_ver()
if vs_ver() == '10.0' and opt.arch_64:
os.environ['DISTUTILS_USE_SDK'] = '1'
os.environ['PATH'] = os.pathsep.join(
[
py_dir(),
os.path.join(py_dir(), 'Scripts'),
r'C:\Program Files\Git\mingw64\bin',
os.environ['PATH'],
]
)
if vs_ver() == '9.0':
logger.info("Fixing VS2008 Express and 64bit builds")
shutil.copyfile(
os.path.join(vc_dir(), r"bin\vcvars64.bat"),
os.path.join(vc_dir(), r"bin\amd64\vcvarsamd64.bat"),
)
# Fix problem with VS2010 Express 64bit missing vcvars64.bat
# Note: repository not cloned at this point, so need to fetch
# file another way
if vs_ver() == '10.0':
if not os.path.exists(
os.path.join(vc_dir(), r"bin\amd64\vcvars64.bat")
):
logger.info("Fixing VS2010 Express and 64bit builds")
with urlopen(
"https://raw.githubusercontent.com/psycopg/psycopg2/"
"master/scripts/vcvars64-vs2010.bat"
) as f:
data = f.read()
with open(
os.path.join(vc_dir(), r"bin\amd64\vcvars64.bat"), 'w'
) as f:
f.write(data)
logger.info("Configuring compiler")
bat_call(
[
os.path.join(vc_dir(), "vcvarsall.bat"),
'x86' if opt.arch_32 else 'amd64',
]
)
def python_info():
logger.info("Python Information")
out = call_command([py_exe(), '--version'], stderr=sp.STDOUT)
logger.info("%s", out)
cmdline = [
py_exe(),
'-c',
"import sys; print('64bit: %s' % (sys.maxsize > 2**32))",
]
out = call_command(cmdline)
logger.info("%s", out)
def step_init():
# The program rc.exe on 64bit with some versions look in the wrong path
# location when building postgresql. This cheats by copying the x64 bit
# files to that location.
if opt.arch_64:
for fn in glob(
r'C:\Program Files\Microsoft SDKs\Windows\v7.0\Bin\x64\rc*'
):
logger.info("Copying %s to a better place" % os.path.basename(fn))
shutil.copy(
fn, r"C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Bin"
)
# Change PostgreSQL config before service starts
logger.info("Configuring Postgres")
with open(os.path.join(pg_dir(), 'data', 'postgresql.conf'), 'a') as f:
# allow > 1 prepared transactions for test cases
print("max_prepared_transactions = 10", file=f)
def bat_call(cmdline):
"""
Simulate 'CALL' from a batch file
Execute CALL *cmdline* and export the changed environment to the current
environment.
nana-nana-nana-nana...
"""
if not isinstance(cmdline, str):
cmdline = ' '.join(c if ' ' not in c else '"%s"' % c for c in cmdline)
pyexe = py_exe()
data = f"""\
CALL {cmdline}
{pyexe} -c "import os, sys, json; json.dump(dict(os.environ), sys.stdout, indent=2)"
"""
logger.debug("preparing file to batcall:\n\n%s", data)
with NamedTemporaryFile(suffix='.bat') as tmp:
fn = tmp.name
with open(fn, "w") as f:
f.write(data)
try:
out = call_command(fn)
# be vewwy vewwy caweful to print the env var as it might contain
# secwet things like your pwecious pwivate key.
# logger.debug("output of command:\n\n%s", out.decode('utf8', 'replace'))
# The output has some useless crap on stdout, because sure, and json
# indented so the last { on column 1 is where we have to start parsing
m = list(re.finditer(b'^{', out, re.MULTILINE))[-1]
out = out[m.start() :]
env = json.loads(out)
for k, v in env.items():
if os.environ.get(k) != v:
logger.info("setting %s=%s", k, v)
os.environ[k] = v
finally:
os.remove(fn)
def py_dir():
"""
Return the path to the target python binary to execute.
"""
dirname = ''.join(
[r"C:\Python", opt.pyver, '-x64' if opt.arch_64 else '']
)
return dirname
def py_exe():
"""
Return the full path of the target python executable.
"""
return os.path.join(py_dir(), 'python.exe')
def vc_dir(vsver=None):
"""
Return the path of the Visual C compiler.
"""
if vsver is None:
vsver = vs_ver()
return r"C:\Program Files (x86)\Microsoft Visual Studio %s\VC" % vsver
def vs_ver(pyver=None):
# Py 2.7 = VS Ver. 9.0 (VS 2008)
# Py 3.4 = VS Ver. 10.0 (VS 2010)
# Py 3.5, 3.6, 3.7 = VS Ver. 14.0 (VS 2015)
if pyver is None:
pyver = opt.pyver
if pyver == '27':
vsver = '9.0'
elif pyver == '34':
vsver = '10.0'
elif pyver in ('35', '36', '37'):
vsver = '14.0'
else:
raise Exception('unexpected python version: %r' % pyver)
return vsver
def pg_dir():
return r"C:\Program Files\PostgreSQL\9.6"
def call_command(cmdline, **kwargs):
logger.debug("calling command: %s", cmdline)
data = sp.check_output(cmdline, **kwargs)
return data
def parse_cmdline():
from argparse import ArgumentParser
parser = ArgumentParser(description=__doc__)
parser.add_argument(
'--pyver',
choices='27 34 35 36 37'.split(),
help="the target python version. Default from PYVER env var",
)
parser.add_argument(
'--pyarch',
choices='32 64'.split(),
help="the target python architecture. Default from PYTHON_ARCH env var",
)
steps = [
n[len(STEP_PREFIX) :]
for n in globals()
if n.startswith(STEP_PREFIX) and callable(globals()[n])
]
parser.add_argument(
'step', choices=steps, help="the appveyor step to execute"
)
opt = parser.parse_args()
# And die if they are not there.
if not opt.pyver:
opt.pyver = os.environ['PYVER']
if not opt.pyarch:
opt.pyarch = os.environ['PYTHON_ARCH']
assert opt.pyarch in ('32', '64')
opt.arch_32 = opt.pyarch == '32'
opt.arch_64 = opt.pyarch == '64'
return opt
if __name__ == '__main__':
sys.exit(main())