From 4e8d21d849dc4c562d02c4aa22683296cbd314f7 Mon Sep 17 00:00:00 2001 From: Jay Berkenbilt Date: Sat, 24 Oct 2020 19:31:09 -0400 Subject: [PATCH] Build Windows releases with openssl; automate external libraries External libraries for Windows are now built automatically in the qpdf/external-libs repository and include openssl in addition to zlib and jpeg. Use these, and update the Windows build to build with the openssl crypto provider by default. We leave the native crypto provider enabled in case there is a problem with openssl and also to continue to exercise that code. --- .github/workflows/main.yml | 15 +++++++ ChangeLog | 6 +++ README-maintainer | 14 ------ README-what-to-download.md | 31 ++++++-------- README-windows.md | 4 +- README.md | 2 +- TODO | 64 ++++++++++++++++++++++------ autofiles.sums | 2 +- build-scripts/build-mac | 7 +-- build-scripts/build-windows | 3 +- build-scripts/download-external-libs | 56 ++++++++++++++++++++++++ build-scripts/make-distfiles | 2 + config-mingw | 2 +- config-msvc | 2 +- configure | 6 ++- configure.ac | 10 +++-- copy_dlls | 1 + 17 files changed, 167 insertions(+), 60 deletions(-) create mode 100755 build-scripts/download-external-libs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ed963ef1..3f86ef9e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,6 +35,11 @@ jobs: with: name: distfiles path: distfiles.zip + - name: 'Upload external libs' + uses: actions/upload-artifact@v1 + with: + name: external-libs + path: external-libs-dist Linux: runs-on: ubuntu-latest steps: @@ -64,6 +69,11 @@ jobs: uses: actions/download-artifact@v2 with: name: distfiles + - name: 'Download external libs' + uses: actions/download-artifact@v2 + with: + name: external-libs + path: . - name: 'Build, test, generate binary distributions' shell: cmd run: build-scripts/build-windows.bat ${{ matrix.wordsize }} ${{ matrix.tool }} @@ -81,6 +91,11 @@ jobs: uses: actions/download-artifact@v2 with: name: distfiles + - name: 'Download external libs' + uses: actions/download-artifact@v2 + with: + name: external-libs + path: . - name: 'Mac build and test' run: build-scripts/build-mac AppImage: diff --git a/ChangeLog b/ChangeLog index fd057636..2cc8ec3e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +2020-10-25 Jay Berkenbilt + + * Official Windows releases are now built using the openssl crypto + provider. The native provider is still available for selection at + runtime using the QPDF_CRYPTO_PROVIDER environment variable. + 2020-10-23 Jay Berkenbilt * Bug fix: when concatenating content streams, insert a newline if diff --git a/README-maintainer b/README-maintainer index d8c0bc73..c4ba1b4a 100644 --- a/README-maintainer +++ b/README-maintainer @@ -326,20 +326,6 @@ To construct a source distribution from a pristine checkout, make build_manual make distclean -To create a source release of external libs, do an export from the -version control system into a directory called `qpdf-external-libs` -and just make a zip file of the result called -`qpdf-external-libs-src.zip`. See the README.txt file there for -information on creating binary external libs releases. Run this from -the external-libs repository: - -git archive --prefix=external-libs/ HEAD . | (cd /tmp; tar xf -) -cd /tmp -zip -r qpdf-external-libs-src.zip external-libs - -When releasing on sourceforge, `external-libs` distributions go in -`external-libs/yyyymmdd`, and qpdf distributions go in `qpdf/vvv`. - For local iteration on the AppImage generation, it works to just ./build-scripts/build-appimage and get the resulting AppImage from the distribution directory. You can also pass -e SKIP_TESTS=1 diff --git a/README-what-to-download.md b/README-what-to-download.md index beede448..205867b2 100644 --- a/README-what-to-download.md +++ b/README-what-to-download.md @@ -1,32 +1,25 @@ To build from source for Linux or other UNIX/UNIX-like systems, it is generally sufficient to download just the source `qpdf-.tar.gz` file. -Virtually all Linux distributions include packages for qpdf. If you'd like to run the latest version of qpdf as an [AppImage](https://appimage.org/), you can download `qpdf--x86_64.AppImage`. This is a self-contained executable that you make symlink `qpdf` to and run on most reasonably recent Linux distributions. See README-appimage.md in the qpdf source distribution for additional details, or run the AppImage with the `--ai-usage` argument to get help specific to the AppImage. +Windows Binaries -For Windows, there are several additional files that you might want to download. +You can download Windiows binaries that are statically linked with qpdf's external dependencies and use the OpenSSL crypto provider. There are several options: -* `qpdf--bin-mingw32.zip` +* `qpdf--bin-mingw32.zip` - 32-bit executables that should work on basically any Windows system, including 64-bit systems. The 32-bit executables are capable of handling files larger than 2 GB. If you just want to use the qpdf command line program or use the qpdf DLL's C-language interface, you can download this file. You can also download this version if you are using MINGW's gcc and want to program using the C++ interface. - If you just want to use the qpdf command line program or use the qpdf DLL's C-language interface, you can download this file. You can also download this version if you are using MINGW's gcc and want to program using the C++ interface. +* `qpdf--bin-mingw64.zip` - A 64-bit version built with mingw. Use this for 64-bit Windows systems. The 32-bit version will also work on Windows 64-bit. Both the 32-bit and the 64-bit version support files over 2 GB in size, but you may find it easier to integrate this with your own software if you use the 64-bit version. -* `qpdf--bin-mingw64.zip` +* `qpdf--bin-msvc32.zip` - If you want to program using qpdf's C++ interface and you are using a recent version of Microsoft Visual C++ in 32-bit mode, you can download this file. - A 64-bit version built with mingw. Use this for 64-bit Windows systems. The 32-bit version will also work on Windows 64-bit. Both the 32-bit and the 64-bit version support files over 2 GB in size, but you may find it easier to integrate this with your own software if you use the 64-bit version. +* `qpdf--bin-msvc64.zip` - If you want to program using qpdf's C++ interface and you are using a recent version of Microsoft Visual C++ in 64-bit mode, you can download this file. -* `qpdf--bin-msvc32.zip` +Linux Binaries - If you want to program using qpdf's C++ interface and you are using Microsoft Visual C++ 2015 in 32-bit mode, you can download this file. +Virtually all Linux distributions include packages for qpdf. There is also a PPA for Ubuntu at https://launchpad.net/~qpdf/+archive/ubuntu/qpdf that includes the latest version of qpdf for recent versions of Ubuntu. However, there are some downloads available for Linux as well. -* `qpdf--bin-msvc64.zip` +* `qpdf--x86_64.AppImage` - If you'd like to run the latest version of qpdf as an [AppImage](https://appimage.org/), you can download this. This is a self-contained executable that you make symlink `qpdf` to and run on most reasonably recent Linux distributions. See README-appimage.md in the qpdf source distribution for additional details, or run the AppImage with the `--ai-usage` argument to get help specific to the AppImage. - If you want to program using qpdf's C++ interface and you are using Microsoft Visual C++ 2015 in 64-bit mode, you can download this file. +* `qpdf--bin-linux-x86_64.zip` - This is a (nearly) stand-alone Linux binary, built using an Ubuntu LTS release. It contains the qpdf executables and shared libraries as well as dependent shared libraries that would not typically be present on a minimal system. This can be used to include qpdf in a minimal environment such as a docker container. It is also known to work as a layer in AWS Lambda and was initially created for that purpose. -* `qpdf-external-libs-bin.zip` - - If you want to build qpdf for Windows yourself with either MINGW or MSVC 2015, you can download this file and extract it inside the qpdf source distribution. Please refer to README-windows.md in the qpdf source distribution for additional details. Note that you need the 2017-08-21 version or later to be able to build qpdf 7.0 or newer. Generally grab the `external-libs` distribution that was the latest version at the time of the release of whichever version of qpdf you are building. - -* `qpdf-external-libs-src.zip` - - If you want to build the external libraries on your own (for Windows or anything else), you can download this archive. In addition to including an unmodified distribution `zlib` and the `jpeg` library, it includes a `README` file and some scripts to help you build it for Windows. You will also have to provide those. - -If you want to build on Windows, please see also README-windows.md in the qpdf source distribution. +Windows Build Support +If you are building on Windows and want to use pre-built external static libraries, you should obtain current versions from https://github.com/qpdf/external-libs/releases. The `external-libs` directory contains older versions that will not work with qpdf versions >= 10.0.2. Please see README-windows.md in the qpdf source distribution. diff --git a/README-windows.md b/README-windows.md index 471e773c..2b182123 100644 --- a/README-windows.md +++ b/README-windows.md @@ -32,11 +32,11 @@ Image comparison tests are disabled by default, but it is possible to run them o # External Libraries -In order to build qpdf, you must have a copy of `zlib` and the `jpeg` library. The easy way to get it is to download the external libs from the qpdf download area. There are packages called `external-libs-bin.zip` and `external-libs-src.zip`. If you are building with MSVC 2015 or MINGW with MSYS2, you can just extract the `qpdf-external-libs-bin.zip` zip file into the top-level qpdf source tree. Note that you need the 2017-08-21 version (at least) to build qpdf 7.0 or greater since this includes jpeg. Passing `--enable-external-libs` to `./configure` (which is done automatically if you follow the instructions below) is sufficient to find them. +In order to build qpdf, you must have a copy of `zlib` and the `jpeg` library. You can download [prebuilt static external libraries from the qpdf/external-libs github repository](https://github.com/qpdf/external-libs/releases). These include `zlib`, `jpeg`, and `openssl` libraries. There are packages called `external-libs-bin.zip` and `external-libs-src.zip`. If you are building with a recent MSVC or MINGW with MSYS2, you can just extract the `qpdf-external-libs-bin.zip` zip file into the top-level qpdf source tree. Note that you need the 2020-10-24 version (at least) to build qpdf 10.0.2 or greater since this includes openssl. Passing `--enable-external-libs` to `./configure` (which is done automatically if you follow the instructions below) is sufficient to find them. You can also obtain `zlib` and `jpeg` directly on your own and install them. If you are using mingw, you can just set `CPPFLAGS`, `LDFLAGS`, and `LIBS` when you run ./configure so that it can find the header files and libraries. If you are building with MSVC and you want to do this, it probably won't work because `./configure` doesn't know how to interpret `LDFLAGS` and `LIBS` properly for MSVC (though qpdf's own build system does). In this case, you can probably get away with cheating by passing `--enable-external-libs` to `./configure` and then just editing `CPPFLAGS`, `LDFLAGS`, `LIBS` in the generated autoconf.mk file. Note that you should use UNIX-like syntax (`-I`, `-L`, `-l`) even though this is not what cl takes on the command line. qpdf's build rules will fix it. -You can also download `qpdf-external-libs-src.zip` and follow the instructions in the README.txt there for how to build external libs. +External libraries are built using GitHub Actions from the [qpdf/external-libs](https://github.com/qpdf/external-libs) repository. # Building from version control diff --git a/README.md b/README.md index 5040645d..784fe091 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Depending on which crypto providers are enabled, then [GnuTLS](https://www.gnutl # Licensing terms of embedded software -QPDF makes use of zlib and jpeg libraries for its functionality. These packages can be downloaded separately from their own download locations, or they can be downloaded in the external-libs area of the qpdf download site. If the optional GnuTLS crypto provider is enabled, then GnuTLS is also required. +QPDF makes use of zlib and jpeg libraries for its functionality. These packages can be downloaded separately from their own download locations. If the optional GnuTLS or OpenSSL crypto providers are enabled, then GnuTLS and/or OpenSSL are also required. Please see the [NOTICE](NOTICE.md) file for information on licenses of embedded software. diff --git a/TODO b/TODO index 242c59c8..3f951ce2 100644 --- a/TODO +++ b/TODO @@ -8,8 +8,6 @@ Candidates for upcoming release * #446: recognize edited QDF files * #436: parsing of document with form xobject -* See if we can work in Windows Build/External Libraries (below) - * QPDFObjectHandle::pipeContentStreams calls finish() after each stream. In some code paths, Pl_Concatenate is used, which suppresses that, but in other code paths, it's not used, and the library relies @@ -94,20 +92,62 @@ GitHub Actions but I'd rather not depend on them. Keep an eye open for this coming to GitHub Actions. -Windows Build/External Libraries -================================ +External Libraries +================== -* Migrate external library build code to a separate repository. +Current state (10.0.2): -* Automate downloading and building latest versions of external - libraries. Add openssl. +* qpdf/external-libs repository builds external-libs on a schedule. + It detects and downloads the latest versions of zlib, jpeg, and + openssl and creates source and binary distribution zip files in an + artifact called "distribution". -* Build external libraries on a schedule and create releases - periodically or when they change. See if we can get rid of the - external-libs branch in qpdf/qpdf. +* Releases in qpdf/external-libs are made manually. They contain + qpdf-external-libs-{bin,src}.zip. + +* The qpdf build finds the latest non-prerelease release and downloads + the qpdf-external-libs-*.zip files from the releases in the setup + stage. + +* To upgrade to a new version of external-libs, create a new release + of qpdf/external-libs (see README-maintainer in external-libs) from + the distribution artifact of the most recent successful build after + ensuring that it works. + +Desired state: + +* The qpdf/external-libs repository should create release candidates. + Ideally, every scheduled run would make its zip files available. + A personal access token with actions:read scope for the + qpdf/external-libs repository is required to download the artifact + from an action run, and qpdf/qpdf's secrets.GITHUB_TOKEN doesn't + have this access. As an alternative, we could have a draft release + in qpdf/external-libs that the qpdf/external-libs build could update + with each candidate. + +* Scheduled runs of the qpdf build in the qpdf/qpdf repository (not a + fork or pull request) could download external-libs from the release + candidate area instead of the latest stable release. Pushes to the + build branch should still use the latest release so it always + matches the main branch. + +* Periodically, we would create a release of external-libs from the + release candidate zip files. This could be done safely because we + know the latest qpdf works with it. This could be done at least + before every release of qpdf, but potientially it could be done at + other times, such as when a new dependency version is available or + after some period of time. + +Other notes: + +* The external-libs branch in qpdf/qpdf was never documented. We might + be able to get away with deleting it. + +* See README-maintainer in qpdf/external-libs for information on + creating a release. This could be at least partially scripted in a + way that works for the qpdf/qpdf repository as well since they are + very similar. -* Update the Windows build so that it uses current versions of - external libraries and openssl as its crypto provider. ABI Changes =========== diff --git a/autofiles.sums b/autofiles.sums index 68d99e36..a29c096a 100644 --- a/autofiles.sums +++ b/autofiles.sums @@ -1,4 +1,4 @@ -cc3c7947646412e7c3152c3ef238226ede1c2199328a38df93debee26184b087 configure.ac +1c8c57be01a03bc85ca49293d0212026e0e9b28a73567bda8e27462abc3ffd1a configure.ac d3f9ee6f6f0846888d9a10fd3dad2e4b1258be84205426cf04d7cef02d61dad7 aclocal.m4 cf2c764639c4c94abc183a0976eca6ae500b80790ea25e3d0af97b23587363b7 libqpdf/qpdf/qpdf-config.h.in 5297971a0ef90bcd5563eb3f7127a032bb76d3ae2af7258bf13479caf8983a60 m4/ax_cxx_compile_stdcxx.m4 diff --git a/build-scripts/build-mac b/build-scripts/build-mac index 96d900bb..eb41f817 100755 --- a/build-scripts/build-mac +++ b/build-scripts/build-mac @@ -1,8 +1,9 @@ #!/bin/bash set -ex -curl -L https://github.com/qpdf/qpdf/raw/external-libs/jpegsrc.v9c.tar.gz -o jpegsrc.v9c.tar.gz -tar xzf jpegsrc.v9c.tar.gz -cd jpeg-9c +cd $(dirname $0)/.. +unzip qpdf-external-libs-src.zip +tar xzf external-libs-src/jpegsrc* +cd jpeg-* ./configure make -k sudo make install diff --git a/build-scripts/build-windows b/build-scripts/build-windows index 2ec6a500..f67b8f2e 100755 --- a/build-scripts/build-windows +++ b/build-scripts/build-windows @@ -23,14 +23,13 @@ fi if [ -f distfiles/distfiles.zip ]; then unzip distfiles/distfiles.zip fi -curl -L https://github.com/qpdf/qpdf/raw/external-libs/qpdf-external-libs-bin.zip -o qpdf-external-libs-bin.zip unzip qpdf-external-libs-bin.zip cwd=`pwd` PATH=$cwd/libqpdf/build:$PATH installdir=install-$tool$wordsize rm -rf $installdir -./config-$tool --enable-show-failed-test-output --disable-crypto-gnutls --disable-crypto-openssl +./config-$tool --enable-show-failed-test-output make -j$(nproc) -k make -k check make install diff --git a/build-scripts/download-external-libs b/build-scripts/download-external-libs new file mode 100755 index 00000000..94d3ff52 --- /dev/null +++ b/build-scripts/download-external-libs @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +import json +import os +import requests +import sys +from zipfile import ZipFile +from operator import itemgetter + + +def warn(*args, **kwargs): + print(*args, **kwargs, file=sys.stderr) + + +def download_file(url, local_filename): + # From https://stackoverflow.com/questions/16694907/ + # download-large-file-in-python-with-requests + if os.path.exists(local_filename): + warn('Using existing', local_filename) + return + warn('Downloading', local_filename) + with requests.get(url, stream=True) as r: + r.raise_for_status() + with open(local_filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + return local_filename + + +bin_name = 'qpdf-external-libs-bin.zip' +src_name = 'qpdf-external-libs-src.zip' +dir_name = 'external-libs-dist' +os.makedirs(dir_name, exist_ok=True) + +r = requests.get( + 'https://api.github.com/repos/qpdf/external-libs/releases') +releases = json.loads(r.text) +by_tag = sorted( + [(r['tag_name'], r) for r in releases + if r['prerelease'] is False], + reverse=True) +latest = by_tag[0][1] +bin_url = None +src_url = None +for i in latest['assets']: + if i['name'] == bin_name: + bin_url = i['browser_download_url'] + elif i['name'] == src_name: + src_url = i['browser_download_url'] +print(bin_url) +download_file(bin_url, f'{dir_name}/{bin_name}') +download_file(src_url, f'{dir_name}/{src_name}') + +print('\n** external library information **') +with ZipFile(f'{dir_name}/{src_name}') as z1: + with z1.open('external-libs-src/versions') as z2: + print(z2.read().decode()) diff --git a/build-scripts/make-distfiles b/build-scripts/make-distfiles index 6a074aee..26ef161d 100755 --- a/build-scripts/make-distfiles +++ b/build-scripts/make-distfiles @@ -1,4 +1,5 @@ #!/bin/bash +cd $(dirname $0)/.. set -ex sudo apt-get update sudo apt-get -y install \ @@ -6,3 +7,4 @@ sudo apt-get -y install \ docbook-xsl fop xsltproc libxml2-utils inkscape imagemagick ./configure --enable-doc-maintenance make -j$(nproc) distfiles.zip +build-scripts/download-external-libs diff --git a/config-mingw b/config-mingw index 4dfe2cf2..c4c89532 100755 --- a/config-mingw +++ b/config-mingw @@ -6,4 +6,4 @@ if g++ -v 2>&1 | grep Target: | grep -q x86_64; then else wordsize=32 fi -./configure --disable-test-compare-images --enable-external-libs --enable-werror --with-buildrules=mingw ${1+"$@"} +./configure --disable-test-compare-images --enable-external-libs --enable-werror --with-buildrules=mingw ${1+"$@"} --enable-crypto-openssl --disable-crypto-gnutls diff --git a/config-msvc b/config-msvc index 36c4aab2..2b648800 100755 --- a/config-msvc +++ b/config-msvc @@ -6,4 +6,4 @@ if cl 2>&1 | grep -q 'for x64'; then else wordsize=32 fi -CC=cl CXX="cl -TP -GR" ./configure --disable-test-compare-images --enable-external-libs --enable-werror --with-buildrules=msvc ${1+"$@"} +CC=cl CXX="cl -TP -GR" ./configure --disable-test-compare-images --enable-external-libs --enable-werror --with-buildrules=msvc ${1+"$@"} --enable-crypto-openssl --disable-crypto-gnutls diff --git a/configure b/configure index 12cf68cc..1c26421b 100755 --- a/configure +++ b/configure @@ -17829,6 +17829,9 @@ $as_echo "#define USE_CRYPTO_NATIVE 1" >>confdefs.h fi +if test "$USE_EXTERNAL_LIBS" = "1"; then + OPENSSL_FOUND=1 +else pkg_failed=no { $as_echo "$as_me:${as_lineno-$LINENO}: checking for pc_openssl" >&5 @@ -17900,6 +17903,7 @@ else $as_echo "yes" >&6; } OPENSSL_FOUND=1 fi +fi if test "$OPENSSL_FOUND" = "0"; then : ac_fn_c_check_header_mongrel "$LINENO" "openssl/evp.h" "ac_cv_header_openssl_evp_h" "$ac_includes_default" @@ -18601,7 +18605,7 @@ if test "$USE_EXTERNAL_LIBS" = "1"; then # much trouble getting it to work with a different compiler. CPPFLAGS="$CPPFLAGS -Iexternal-libs/include" LDFLAGS="$LDFLAGS -Lexternal-libs/lib-$BUILDRULES$WINDOWS_WORDSIZE" - LIBS="$LIBS -lz -ljpeg" + LIBS="$LIBS -lz -ljpeg -lssl -lcrypto -lmsvcrt -lws2_32 -lshell32 -ladvapi32 -lgdi32 -luser32 -lcrypt32" fi cat >confcache <<\_ACEOF diff --git a/configure.ac b/configure.ac index 136e51b7..22a699a7 100644 --- a/configure.ac +++ b/configure.ac @@ -608,8 +608,12 @@ dnl If the openssl provider is not explicitly disabled, enable it if dnl openssl is available. If the openssl provider is explicitly dnl disabled, do not link with openssl even if present. -PKG_CHECK_MODULES([pc_openssl], [openssl >= 1.1.0], - [OPENSSL_FOUND=1], [OPENSSL_FOUND=0]) +if test "$USE_EXTERNAL_LIBS" = "1"; then + OPENSSL_FOUND=1 +else + PKG_CHECK_MODULES([pc_openssl], [openssl >= 1.1.0], + [OPENSSL_FOUND=1], [OPENSSL_FOUND=0]) +fi dnl Override pkg-config if headers and libraries are present. AS_IF([test "$OPENSSL_FOUND" = "0"], @@ -958,7 +962,7 @@ if test "$USE_EXTERNAL_LIBS" = "1"; then # much trouble getting it to work with a different compiler. CPPFLAGS="$CPPFLAGS -Iexternal-libs/include" LDFLAGS="$LDFLAGS -Lexternal-libs/lib-$BUILDRULES$WINDOWS_WORDSIZE" - LIBS="$LIBS -lz -ljpeg" + LIBS="$LIBS -lz -ljpeg -lssl -lcrypto -lmsvcrt -lws2_32 -lshell32 -ladvapi32 -lgdi32 -luser32 -lcrypt32" fi AC_OUTPUT diff --git a/copy_dlls b/copy_dlls index 8511abce..72fe95a0 100644 --- a/copy_dlls +++ b/copy_dlls @@ -98,6 +98,7 @@ sub get_dlls my $dll = $1; $dll =~ tr/A-Z/a-z/; next if $dll =~ m/^(kernel32|user32|msvcrt|advapi32)\.dll$/; + next if $dll =~ m/^(api-ms-win.*|ws2_32|crypt32|bcrypt)\.dll$/; push(@result, $dll); } elsif (m/^Magic.*\((PE.+?)\)/)