curl/curl
1
1
Fork 0
mirror of https://github.com/curl/curl.git synced 2025-09-13 23:52:42 +03:00
Code Issues Packages Projects Releases Wiki Activity

dist: set -eu, fix shellcheck, make reproducible and smaller tarballs

Browse Source 
- set bash `-eu` and fix fallouts.
- fix shellcheck warnings.
- set and use `SOURCE_DATE_EPOCH` for reproducibility.
  Authored-by: Daniel J. H.
  Ref: #13280
- set `TZ=UTC` and `LC_ALL=C` for reproducibility.
- make file timestamps in tarball/zip reproducible.
- make directory timestamps in zip reproducible.
- make timestamps of tarballs/zip reproducible.
- make file order in tarball/zip reproducible.
- omit extra file metadata from zip for reproducibility.
- use maximum zip compression.
- use POSIX `ustar` tarball format to avoid supply chain vulnerability:
  https://seclists.org/oss-sec/2021/q4/0
- make uid/gid in tarball reproducible.
- omit owner user/group names from tarball for reproducibility and privacy.
- omit current timestamp from .gz header for reproducibility.
- display SHA-256 hashes of produced tarballs/zip.
- fix whitespace.

`.tar.gz` also became smaller in the process: 4,462,311 -> 4,148,249 bytes (8.7.1)

Requires GNU tar, GNU date, `sha256sum`.

Reviewed-by: Daniel Stenberg
Ref: #13250
Closes #13299
This commit is contained in:
Viktor Szakats 2024-04-05 13:27:05 +00:00
parent a42de088a2
commit 860cd5fc2d
No known key found for this signature in database
GPG Key ID: B5ABD165E2AEF201
1 changed files with 68 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
maketgz
View File

@ -26,26 +26,33 @@
# #
########################################################################### ###########################################################################
version=$1 set -eu
export LC_ALL=C
export TZ=UTC
version="${1:-}"
if [ -z "$version" ]; then if [ -z "$version" ]; then
echo "Specify a version number!" echo "Specify a version number!"
exit exit
fi fi
if [ "xonly" = "x$2" ]; then if [ "only" = "${2:-}" ]; then
echo "Setup version number only!" echo "Setup version number only!"
only=1 only=1
else
only=
fi fi
libversion="$version" libversion="$version"
# we make curl the same version as libcurl # we make curl the same version as libcurl
curlversion=$libversion curlversion="$libversion"
major=`echo $libversion | cut -d. -f1 | sed -e "s/[^0-9]//g"` major=$(echo "$libversion" | cut -d. -f1 | sed -e "s/[^0-9]//g")
minor=`echo $libversion | cut -d. -f2 | sed -e "s/[^0-9]//g"` minor=$(echo "$libversion" | cut -d. -f2 | sed -e "s/[^0-9]//g")
patch=`echo $libversion | cut -d. -f3 | cut -d- -f1 | sed -e "s/[^0-9]//g"` patch=$(echo "$libversion" | cut -d. -f3 | cut -d- -f1 | sed -e "s/[^0-9]//g")
if test -z "$patch"; then if test -z "$patch"; then
echo "invalid version number? needs to be z.y.z" echo "invalid version number? needs to be z.y.z"
@ -59,7 +66,8 @@ fi
echo "removing all old *.dist files" echo "removing all old *.dist files"
find . -name "*.dist" -a ! -name Makefile.dist -exec rm {} \; find . -name "*.dist" -a ! -name Makefile.dist -exec rm {} \;
numeric=`perl -e 'printf("%02x%02x%02x\n", '"$major, $minor, $patch);"` numeric=$(perl -e \
'printf("%02x%02x%02x\n", '"$major, $minor, $patch);")
HEADER=include/curl/curlver.h HEADER=include/curl/curlver.h
CHEADER=src/tool_version.h CHEADER=src/tool_version.h
@ -70,37 +78,39 @@ if test -z "$only"; then
ext=".dist" ext=".dist"
# when not setting up version numbers locally # when not setting up version numbers locally
for a in $HEADER $CHEADER $PLIST; do for a in $HEADER $CHEADER $PLIST; do
cp $a "$a$ext" cp "$a" "$a$ext"
done done
HEADER="$HEADER$ext" HEADER="$HEADER$ext"
CHEADER="$CHEADER$ext" CHEADER="$CHEADER$ext"
PLIST="$PLIST$ext" PLIST="$PLIST$ext"
fi fi
# requires a date command that knows + for format # requires a date command that knows + for format and -d for date input
datestamp=`date +"%F"` timestamp=${SOURCE_DATE_EPOCH:-$(date +"%s")}
datestamp=$(date -d "@$timestamp" +"%F")
filestamp=$(date -d "@$timestamp" +"%Y%m%d%H%M.%S")
# Replace version number in header file: # Replace version number in header file:
sed -i.bak \ sed -i.bak \
-e 's/^#define LIBCURL_VERSION .*/#define LIBCURL_VERSION "'$libversion'"/g' \ -e "s/^#define LIBCURL_VERSION .*/#define LIBCURL_VERSION \"$libversion\"/g" \
-e 's/^#define LIBCURL_VERSION_NUM .*/#define LIBCURL_VERSION_NUM 0x'$numeric'/g' \ -e "s/^#define LIBCURL_VERSION_NUM .*/#define LIBCURL_VERSION_NUM 0x$numeric/g" \
-e 's/^#define LIBCURL_VERSION_MAJOR .*/#define LIBCURL_VERSION_MAJOR '$major'/g' \ -e "s/^#define LIBCURL_VERSION_MAJOR .*/#define LIBCURL_VERSION_MAJOR $major/g" \
-e 's/^#define LIBCURL_VERSION_MINOR .*/#define LIBCURL_VERSION_MINOR '$minor'/g' \ -e "s/^#define LIBCURL_VERSION_MINOR .*/#define LIBCURL_VERSION_MINOR $minor/g" \
-e 's/^#define LIBCURL_VERSION_PATCH .*/#define LIBCURL_VERSION_PATCH '$patch'/g' \ -e "s/^#define LIBCURL_VERSION_PATCH .*/#define LIBCURL_VERSION_PATCH $patch/g" \
-e "s/^#define LIBCURL_TIMESTAMP .*/#define LIBCURL_TIMESTAMP \"$datestamp\"/g" \ -e "s/^#define LIBCURL_TIMESTAMP .*/#define LIBCURL_TIMESTAMP \"$datestamp\"/g" \
$HEADER "$HEADER"
rm -f "$HEADER.bak" rm -f "$HEADER.bak"
# Replace version number in header file: # Replace version number in header file:
sed -i.bak 's/#define CURL_VERSION .*/#define CURL_VERSION "'$curlversion'"/g' $CHEADER sed -i.bak "s/#define CURL_VERSION .*/#define CURL_VERSION \"$curlversion\"/g" "$CHEADER"
rm -f "$CHEADER.bak" rm -f "$CHEADER.bak"
# Replace version number in plist file: # Replace version number in plist file:
sed "s/@CURL_PLIST_VERSION@/$curlversion/g" < $PLISTO.in >$PLIST sed "s/@CURL_PLIST_VERSION@/$curlversion/g" < "$PLISTO.in" > "$PLIST"
if test -n "$only"; then if test -n "$only"; then
# done! # done!
exit; exit
fi fi
echo "curl version $curlversion" echo "curl version $curlversion"
@ -110,7 +120,7 @@ echo "datestamp $datestamp"
findprog() { findprog() {
file="$1" file="$1"
for part in `echo $PATH| tr ':' ' '`; do for part in $(echo "$PATH" | tr ':' ' '); do
path="$part/$file" path="$part/$file"
if [ -x "$path" ]; then if [ -x "$path" ]; then
# there it is! # there it is!
@ -137,7 +147,7 @@ echo "Re-running config.status"
# #
if { findprog automake >/dev/null 2>/dev/null; } then if { findprog automake >/dev/null 2>/dev/null; } then
echo "- Could not find or run automake, I hope you know what you're doing!" echo "- Could not find or run automake, I hope you know what you are doing!"
else else
echo "Runs automake --include-deps" echo "Runs automake --include-deps"
automake --include-deps Makefile >/dev/null automake --include-deps Makefile >/dev/null
@ -168,7 +178,7 @@ echo "produce RELEASE-TOOLS.md"
echo "make dist" echo "make dist"
targz="curl-$version.tar.gz" targz="curl-$version.tar.gz"
make -sj dist VERSION=$version make -sj dist "VERSION=$version"
res=$? res=$?
if test "$res" != 0; then if test "$res" != 0; then
@ -176,6 +186,23 @@ if test "$res" != 0; then
exit 2 exit 2
fi fi
retar() {
tempdir=$1
rm -rf "$tempdir"
mkdir "$tempdir"
cd "$tempdir"
gzip -dc "../$targz" | tar -xf -
find curl-* -type f -exec touch -c -t "$filestamp" '{}' +
find curl-* -type f | sort | tar --create --format=ustar --owner=0 --group=0 --numeric-owner --files-from - | gzip --best --no-name > out.tar.gz
mv out.tar.gz ../
cd ..
rm -rf "$tempdir"
}
retar ".tarbuild"
echo "replace $targz with out.tar.gz"
mv out.tar.gz "$targz"
############################################################################ ############################################################################
# #
# Now make a bz2 archive from the tar.gz original # Now make a bz2 archive from the tar.gz original
@ -183,7 +210,7 @@ fi
bzip2="curl-$version.tar.bz2" bzip2="curl-$version.tar.bz2"
echo "Generating $bzip2" echo "Generating $bzip2"
gzip -dc $targz | bzip2 --best > $bzip2 gzip -dc "$targz" | bzip2 --best > "$bzip2"
############################################################################ ############################################################################
# #
@ -192,21 +219,22 @@ gzip -dc $targz | bzip2 --best > $bzip2
xz="curl-$version.tar.xz" xz="curl-$version.tar.xz"
echo "Generating $xz" echo "Generating $xz"
gzip -dc $targz | xz -6e - > $xz gzip -dc "$targz" | xz -6e - > "$xz"
############################################################################ ############################################################################
# #
# Now make a zip archive from the tar.gz original # Now make a zip archive from the tar.gz original
# #
makezip() { makezip() {
rm -rf $tempdir rm -rf "$tempdir"
mkdir $tempdir mkdir "$tempdir"
cd $tempdir cd "$tempdir"
gzip -dc ../$targz | tar -xf - gzip -dc "../$targz" | tar -xf -
find . | zip $zip -@ >/dev/null find . -depth -type d -exec touch -c -t "$filestamp" '{}' +
mv $zip ../ find . | sort | zip -9 -X "$zip" -@ >/dev/null
mv "$zip" ../
cd .. cd ..
rm -rf $tempdir rm -rf "$tempdir"
} }
zip="curl-$version.zip" zip="curl-$version.zip"
@ -214,10 +242,14 @@ echo "Generating $zip"
tempdir=".builddir" tempdir=".builddir"
makezip makezip
# Set deterministic timestamp
touch -c -t "$filestamp" "$targz" "$bzip2" "$xz" "$zip"
echo "------------------" echo "------------------"
echo "maketgz report:" echo "maketgz report:"
echo "" echo ""
ls -l $targz $bzip2 $zip $xz ls -l "$targz" "$bzip2" "$xz" "$zip"
sha256sum "$targz" "$bzip2" "$xz" "$zip"
echo "Run this:" echo "Run this:"
echo "gpg -b -a $targz && gpg -b -a $bzip2 && gpg -b -a $zip && gpg -b -a $xz" echo "gpg -b -a '$targz' && gpg -b -a '$bzip2' && gpg -b -a '$xz' && gpg -b -a '$zip'"