From c87519743222e9579438504c729086c827a4df42 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 15 Apr 2019 00:38:51 +0100 Subject: [PATCH] Build libpq from Python --- .appveyor.yml | 54 --------- scripts/appveyor.py | 219 +++++++++++++++++++++++++++++------ scripts/win_openssl_11.patch | 20 ---- 3 files changed, 186 insertions(+), 107 deletions(-) delete mode 100644 scripts/win_openssl_11.patch diff --git a/.appveyor.yml b/.appveyor.yml index 6de23903..8ad507e5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -85,60 +85,6 @@ init: install: - "%PYEXE% C:\\appveyor.py install" - # Setup directories for building PostgreSQL librarires - - ECHO ******************************************************************* - - ECHO Preparing for building PostgreSQL libraries - - ECHO ******************************************************************* - - SET PGTOP=%BASE_DIR%\postgresql - - IF NOT EXIST %PGTOP%\include MKDIR %PGTOP%\include - - IF NOT EXIST %PGTOP%\lib MKDIR %PGTOP%\lib - - IF NOT EXIST %PGTOP%\bin MKDIR %PGTOP%\bin - - # Download PostgreSQL source - - CD C:\Others - - IF NOT EXIST postgres-REL_%POSTGRES_VERSION%.zip ( - curl -fsSL -o postgres-REL_%POSTGRES_VERSION%.zip https://github.com/postgres/postgres/archive/REL_%POSTGRES_VERSION%.zip - ) - - # Setup build config file (config.pl) - # Hack the Mkvcbuild.pm file so we build the lib version of libpq - # Build libpgport, libpgcommon, libpq - # Install includes - # Copy over built libraries - # Prepare local include directory for building from - # Build pg_config in place - # Note patch for OpenSSL 1.1 configuration. See: - # https://www.postgresql-archive.org/Compile-psql-9-6-with-SSL-Version-1-1-0-td6054118.html - # NOTE: Cannot set and use the same variable inside an IF - - SET PGBUILD=%BUILD_DIR%\postgres-REL_%POSTGRES_VERSION% - - IF NOT EXIST %PGTOP%\lib\libpq.lib ( - CD %BUILD_DIR% && - 7z x C:\Others\postgres-REL_%POSTGRES_VERSION%.zip && - CD postgres-REL_%POSTGRES_VERSION% && - patch -p1 < %APPVEYOR_BUILD_FOLDER%\scripts\win_openssl_11.patch && - CD src\tools\msvc && - ECHO $config-^>{ldap} = 0; > config.pl && - ECHO $config-^>{openssl} = "%OPENSSLTOP:\=\\%"; >> config.pl && - ECHO.>> config.pl && - ECHO 1;>> config.pl && - perl -pi.bak -e "s/'libpq', 'dll'/'libpq', 'lib'/g" Mkvcbuild.pm && - build libpgport && - build libpgcommon && - build libpq && - ECHO "" > %PGBUILD%\src\backend\parser\gram.h && - perl -pi.bak -e "s/qw\(Install\)/qw\(Install CopyIncludeFiles\)/g" Install.pm && - perl -MInstall=CopyIncludeFiles -e"chdir('../../..'); CopyIncludeFiles('%PGTOP%')" && - COPY %PGBUILD%\Release\libpgport\libpgport.lib %PGTOP%\lib && - COPY %PGBUILD%\Release\libpgcommon\libpgcommon.lib %PGTOP%\lib && - COPY %PGBUILD%\Release\libpq\libpq.lib %PGTOP%\lib && - XCOPY /Y /S %PGBUILD%\src\include\port\win32\* %PGBUILD%\src\include && - XCOPY /Y /S %PGBUILD%\src\include\port\win32_msvc\* %PGBUILD%\src\include && - CD %PGBUILD%\src\bin\pg_config && - cl pg_config.c /MT /nologo /I%PGBUILD%\src\include /link /LIBPATH:%PGTOP%\lib libpgcommon.lib libpgport.lib advapi32.lib /NODEFAULTLIB:libcmt.lib /OUT:%PGTOP%\bin\pg_config.exe && - CD %BASE_DIR% && - RMDIR /S /Q %PGBUILD% - ) - build: off #before_build: diff --git a/scripts/appveyor.py b/scripts/appveyor.py index 688519a3..4ed758d0 100755 --- a/scripts/appveyor.py +++ b/scripts/appveyor.py @@ -95,15 +95,13 @@ def setup_env(): def python_info(): logger.info("Python Information") - out = call_command([py_exe(), '--version'], stderr=sp.STDOUT) + out = out_command([py_exe(), '--version'], stderr=sp.STDOUT) logger.info("%s", out) - cmdline = [ - py_exe(), - '-c', - "import sys; print('64bit: %s' % (sys.maxsize > 2**32))", - ] - out = call_command(cmdline) + out = out_command( + [py_exe(), '-c'] + + ["import sys; print('64bit: %s' % (sys.maxsize > 2**32))"] + ) logger.info("%s", out) @@ -128,12 +126,19 @@ def step_init(): def step_install(): - build_openssl() + # TODO: enable again + # build_openssl() + build_libpq() def build_openssl(): - # Setup directories for building OpenSSL libraries top = os.path.join(base_dir(), 'openssl') + if os.path.exists(os.path.join(top, 'lib', 'libssl.lib')): + return + + logger.info("Building OpenSSL") + + # Setup directories for building OpenSSL libraries ensure_dir(os.path.join(top, 'include', 'openssl')) ensure_dir(os.path.join(top, 'lib')) @@ -156,27 +161,16 @@ def build_openssl(): f"https://github.com/openssl/openssl/archive/{zipname}", zipfile ) - if os.path.exists(os.path.join(top, 'lib', 'libssl.lib')): - return - with ZipFile(zipfile) as z: z.extractall(path=build_dir()) os.chdir(os.path.join(build_dir(), f"openssl-OpenSSL_{ver}")) - cmdline = [ - 'perl', - 'Configure', - target, - 'no-asm', - 'no-shared', - 'no-zlib', - f'--prefix={top}', - f'--openssldir={top}', - ] - call_command(cmdline, output=False) + run_command( + ['perl', 'Configure', target, 'no-asm'] + + ['no-shared', 'no-zlib', f'--prefix={top}', f'--openssldir={top}'] + ) - cmdline = "nmake build_libs install_dev".split() - call_command(cmdline, output=False) + run_command("nmake build_libs install_dev".split()) assert os.path.exists(os.path.join(top, 'lib', 'libssl.lib')) @@ -184,8 +178,115 @@ def build_openssl(): shutil.rmtree(os.path.join(build_dir(), f"openssl-OpenSSL_{ver}")) +def build_libpq(): + top = os.path.join(base_dir(), 'postgresql') + if os.path.exists(os.path.join(top, 'lib', 'libpq.lib')): + return + + logger.info("Building libpq") + + # Setup directories for building PostgreSQL librarires + ensure_dir(os.path.join(top, 'include')) + ensure_dir(os.path.join(top, 'lib')) + ensure_dir(os.path.join(top, 'bin')) + + ver = os.environ['POSTGRES_VERSION'] + + # Download PostgreSQL source + zipname = f'postgres-REL_{ver}.zip' + zipfile = os.path.join(r'C:\Others', zipname) + if not os.path.exists(zipfile): + download( + f"https://github.com/postgres/postgres/archive/REL_{ver}.zip", + zipfile, + ) + + with ZipFile(zipfile) as z: + z.extractall(path=build_dir()) + + pgbuild = os.path.join(build_dir(), f"postgres-REL_{ver}") + os.chdir(pgbuild) + + # Patch for OpenSSL 1.1 configuration. See: + # https://www.postgresql-archive.org/Compile-psql-9-6-with-SSL-Version-1-1-0-td6054118.html + assert os.path.exists("src/include/pg_config.h.win32") + with open("src/include/pg_config.h.win32", 'a') as f: + print( + """ +#define HAVE_ASN1_STRING_GET0_DATA 1 +#define HAVE_BIO_GET_DATA 1 +#define HAVE_BIO_METH_NEW 1 +#define HAVE_OPENSSL_INIT_SSL 1 +""", + file=f, + ) + + # Setup build config file (config.pl) + os.chdir("src/tools/msvc") + with open("config.pl", 'w') as f: + print( + """\ +$config->{ldap} = 0; +$config->{openssl} = "%s"; + +1; +""" + % os.path.join(base_dir(), 'openssl').replace('\\', '\\\\'), + file=f, + ) + + # Hack the Mkvcbuild.pm file so we build the lib version of libpq + file_replace('Mkvcbuild.pm', "'libpq', 'dll'", "'libpq', 'lib'") + + # Build libpgport, libpgcommon, libpq + run_command([which("build"), "libpgport"]) + run_command([which("build"), "libpgcommon"]) + run_command([which("build"), "libpq"]) + + # Install includes + with open(os.path.join(pgbuild, "src/backend/parser/gram.h"), "w") as f: + print("", file=f) + + # Copy over built libraries + file_replace("Install.pm", "qw(Install)", "qw(Install CopyIncludeFiles)") + run_command( + ["perl", "-MInstall=CopyIncludeFiles", "-e"] + + [f"chdir('../../..'); CopyIncludeFiles('{top}')"] + ) + + for lib in ('libpgport', 'libpgcommon', 'libpq'): + shutil.copy( + os.path.join(pgbuild, f'Release/{lib}/{lib}.lib'), + os.path.join(top, 'lib'), + ) + + # Prepare local include directory for building from + for dir in ('win32', 'win32_msvc'): + merge_dir( + os.path.join(pgbuild, f"src/include/port/{dir}"), + os.path.join(pgbuild, "src/include"), + ) + + # Build pg_config in place + os.chdir(os.path.join(pgbuild, 'src/bin/pg_config')) + run_command( + ['cl', 'pg_config.c', '/MT', '/nologo', fr'/I{pgbuild}\src\include'] + + ['/link', fr'/LIBPATH:{top}\lib'] + + ['libpgcommon.lib', 'libpgport.lib', 'advapi32.lib'] + + ['/NODEFAULTLIB:libcmt.lib'] + + [fr'/OUT:{top}\bin\pg_config.exe'] + ) + + assert os.path.exists(os.path.join(top, 'lib', 'libpq.lib')) + assert os.path.exists(os.path.join(top, 'bin', 'pg_config.exe')) + + os.chdir(base_dir()) + shutil.rmtree(os.path.join(pgbuild)) + + def download(url, fn): """Download a file locally""" + logger.info("downloading %s", url) with open(fn, 'wb') as fo, urlopen(url) as fi: while 1: data = fi.read(8192) @@ -193,6 +294,37 @@ def download(url, fn): break fo.write(data) + logger.info("file downloaded: %s", fn) + + +def file_replace(fn, s1, s2): + """ + Replace all the occurrences of the string s1 into s2 in the file fn. + """ + assert os.path.exists(fn) + with open(fn, 'r+') as f: + data = f.read() + f.seek(0) + f.write(data.replace(s1, s2)) + f.truncate() + + +def merge_dir(src, tgt): + """ + Merge the content of the directory src into the directory tgt + + Reproduce the semantic of "XCOPY /Y /S src/* tgt" + """ + for dp, _dns, fns in os.walk(src): + logger.debug("dirpath %s", dp) + if not fns: + continue + assert dp.startswith(src) + subdir = dp[len(src) :].lstrip(os.sep) + tgtdir = ensure_dir(os.path.join(tgt, subdir)) + for fn in fns: + shutil.copy(os.path.join(dp, fn), tgtdir) + def bat_call(cmdline): """ @@ -223,7 +355,7 @@ CALL {cmdline} f.write(data) try: - out = call_command(fn) + out = out_command(fn) # be vewwy vewwy caweful to print the env var as it might contain # secwet things like your pwecious pwivate key. # logger.debug("output of command:\n\n%s", out.decode('utf8', 'replace')) @@ -307,20 +439,41 @@ def ensure_dir(dir): return dir -def call_command(cmdline, output=True, **kwargs): +def run_command(cmdline, **kwargs): logger.debug("calling command: %s", cmdline) - if output: - data = sp.check_output(cmdline, **kwargs) - return data - else: - sp.check_call(cmdline, **kwargs) + sp.check_call(cmdline, **kwargs) + + +def out_command(cmdline, **kwargs): + logger.debug("calling command: %s", cmdline) + data = sp.check_output(cmdline, **kwargs) + return data def setenv(k, v): - logger.info("setting %s=%s", k, v) + logger.debug("setting %s=%s", k, v) os.environ[k] = v +def which(name): + """ + Return the full path of a command found on the path + """ + base, ext = os.path.splitext(name) + if not ext: + exts = ('.com', '.exe', '.bat', '.cmd') + else: + exts = (ext,) + + for dir in ['.'] + os.environ['PATH'].split(os.pathsep): + for ext in exts: + fn = os.path.join(dir, base + ext) + if os.path.isfile(fn): + return fn + + raise Exception("couldn't find program on path: %s" % name) + + def parse_cmdline(): from argparse import ArgumentParser diff --git a/scripts/win_openssl_11.patch b/scripts/win_openssl_11.patch deleted file mode 100644 index 57d0bd81..00000000 --- a/scripts/win_openssl_11.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/src/include/pg_config.h.win32 b/src/include/pg_config.h.win32 -index dfd6972383..72d1259ffb 100644 ---- a/src/include/pg_config.h.win32 -+++ b/src/include/pg_config.h.win32 -@@ -74,12 +74,15 @@ - - /* Define to 1 if you have the `ASN1_STRING_get0_data' function. */ - /* #undef HAVE_ASN1_STRING_GET0_DATA */ -+#define HAVE_ASN1_STRING_GET0_DATA 1 - - /* Define to 1 if you have the `BIO_get_data' function. */ - /* #undef HAVE_BIO_GET_DATA */ -+#define HAVE_BIO_GET_DATA 1 - - /* Define to 1 if you have the `BIO_meth_new' function. */ - /* #undef HAVE_BIO_METH_NEW */ -+#define HAVE_BIO_METH_NEW 1 - - /* Define to 1 if you have the `cbrt' function. */ - //#define HAVE_CBRT 1