Build openssl from Python

.appveyor.yml

@@ -83,51 +83,7 @@ init:
 
 # Repository gets cloned, Cache is restored
 install:
-    # We start off CD'ed to cloned folder
-    - SET BASE_DIR=C:\Others\%PYTHON_ARCH%\%VS_VER%
-    - SET BUILD_DIR=%BASE_DIR%\Builds
-    - IF NOT EXIST %BUILD_DIR% MKDIR %BUILD_DIR%
-
-    - ECHO *******************************************************************
-    - ECHO Initialized variables specific for this build
-    - ECHO *******************************************************************
-    - ECHO %BASE_DIR%
-    - ECHO %BUILD_DIR%
-    - ECHO *******************************************************************
-
-    # Setup directories for building OpenSSL libraries
-    - ECHO *******************************************************************
-    - ECHO Preparing for building OpenSSL
-    - ECHO *******************************************************************
-    - SET OPENSSLTOP=%BASE_DIR%\openssl
-    - IF NOT EXIST %OPENSSLTOP%\include\openssl MKDIR %OPENSSLTOP%\include\openssl
-    - IF NOT EXIST %OPENSSLTOP%\lib MKDIR %OPENSSLTOP%\lib
-
-    # Setup OpenSSL Environment Variables based on processor architecture
-    - ps: >-
-        If ($env:PYTHON_ARCH -Match "32" ) {
-            $env:VCVARS_PLATFORM="x86"
-            $env:TARGET="VC-WIN32"
-        } Else {
-            $env:VCVARS_PLATFORM="amd64"
-            $env:TARGET="VC-WIN64A"
-            $env:CPU="AMD64"
-        }
-    # Download OpenSSL source
-    - CD C:\Others
-    - IF NOT EXIST OpenSSL_%OPENSSL_VERSION%.zip (
-        curl -fsSL -o OpenSSL_%OPENSSL_VERSION%.zip https://github.com/openssl/openssl/archive/OpenSSL_%OPENSSL_VERSION%.zip
-      )
-
-    - IF NOT EXIST %OPENSSLTOP%\lib\libssl.lib (
-        CD %BUILD_DIR% &&
-        7z x C:\Others\OpenSSL_%OPENSSL_VERSION%.zip &&
-        CD openssl-OpenSSL_%OPENSSL_VERSION% &&
-        perl Configure %TARGET% no-asm no-shared no-zlib --prefix=%OPENSSLTOP% --openssldir=%OPENSSLTOP% &&
-        nmake build_libs install_dev &&
-        CD %BASE_DIR% &&
-        RMDIR /S /Q %BUILD_DIR%\openssl-OpenSSL_%OPENSSL_VERSION%
-      )
+    - "%PYEXE% C:\\appveyor.py install"
 
     # Setup directories for building PostgreSQL librarires
     - ECHO *******************************************************************

scripts/appveyor.py

@@ -15,9 +15,10 @@ import shutil
 import logging
 import subprocess as sp
 from glob import glob
-from urllib.request import urlopen
+from zipfile import ZipFile
 from tempfile import NamedTemporaryFile
 from functools import lru_cache
+from urllib.request import urlopen
 
 opt = None
 STEP_PREFIX = 'step_'
@@ -126,6 +127,73 @@ def step_init():
         print("max_prepared_transactions = 10", file=f)
 
 
+def step_install():
+    build_openssl()
+
+
+def build_openssl():
+    # Setup directories for building OpenSSL libraries
+    top = os.path.join(base_dir(), 'openssl')
+    ensure_dir(os.path.join(top, 'include', 'openssl'))
+    ensure_dir(os.path.join(top, 'lib'))
+
+    # Setup OpenSSL Environment Variables based on processor architecture
+    if opt.arch_32:
+        target = 'VC-WIN32'
+        setenv('VCVARS_PLATFORM', 'x86')
+    else:
+        target = 'VC-WIN64A'
+        setenv('VCVARS_PLATFORM', 'amd64')
+        setenv('CPU', 'AMD64')
+
+    ver = os.environ['OPENSSL_VERSION']
+
+    # Download OpenSSL source
+    zipname = f'OpenSSL_{ver}.zip'
+    zipfile = os.path.join(r'C:\Others', zipname)
+    if not os.path.exists(zipfile):
+        download(
+            f"https://github.com/openssl/openssl/archive/{zipname}", zipfile
+        )
+
+    if os.path.exists(os.path.join(top, 'lib', 'libssl.lib')):
+        return
+
+    with ZipFile(zipfile) as z:
+        z.extractall(path=build_dir())
+
+    os.chdir(os.path.join(build_dir(), f"openssl-OpenSSL_{ver}"))
+    cmdline = [
+        'perl',
+        'Configure',
+        target,
+        'no-asm',
+        'no-shared',
+        'no-zlib',
+        f'--prefix={top}',
+        f'--openssldir={top}',
+    ]
+    call_command(cmdline, output=False)
+
+    cmdline = "nmake build_libs install_dev".split()
+    call_command(cmdline, output=False)
+
+    assert os.path.exists(os.path.join(top, 'lib', 'libssl.lib'))
+
+    os.chdir(base_dir())
+    shutil.rmtree(os.path.join(build_dir(), f"openssl-OpenSSL_{ver}"))
+
+
+def download(url, fn):
+    """Download a file locally"""
+    with open(fn, 'wb') as fo, urlopen(url) as fi:
+        while 1:
+            data = fi.read(8192)
+            if not data:
+                break
+            fo.write(data)
+
+
 def bat_call(cmdline):
     """
     Simulate 'CALL' from a batch file
@@ -221,10 +289,31 @@ def pg_dir():
     return r"C:\Program Files\PostgreSQL\9.6"
 
 
-def call_command(cmdline, **kwargs):
+def base_dir():
+    rv = r"C:\Others\%s\%s" % (opt.pyarch, vs_ver())
+    return ensure_dir(rv)
+
+
+def build_dir():
+    rv = os.path.join(base_dir(), 'Builds')
+    return ensure_dir(rv)
+
+
+def ensure_dir(dir):
+    if not os.path.exists(dir):
+        logger.info("creating directory %s", dir)
+        os.makedirs(dir)
+
+    return dir
+
+
+def call_command(cmdline, output=True, **kwargs):
     logger.debug("calling command: %s", cmdline)
+    if output:
         data = sp.check_output(cmdline, **kwargs)
         return data
+    else:
+        sp.check_call(cmdline, **kwargs)
 
 
 def setenv(k, v):