Merge pull request #33603 from sjorge/2016.3-smartos-esky

allow esky packages to be build on base64 2015Q4

pkg/smartos/esky/BUILD_ENVIRONMENT.md

@@ -6,58 +6,50 @@ However it does document pretty thoroughly how I initially created a build envir
 for packaging up esky builds for SmartOS
 
 ```bash
-#!/bin/bash
+#!/bin/sh
+set -ux
 
-export PATH=$PATH:/opt/local/gcc47/bin/
-HERE=$(pwd)
+## environment
+export PATH=$PATH:/opt/local/gcc49/bin/
+BLDPATH=/tmp/bbfreeze_loader
+SALTBASE=/data
 
-mv /opt/local /opt/local.backup ; hash -r
-cd /
-curl http://pkgsrc.joyent.com/packages/SmartOS/bootstrap/bootstrap-2014Q4-x86_64.tar.gz | gtar xz
-hash -r
-
-rm -rf /var/db/pkgin/
-pkgin -y up
+## packages
 pkgin -y in build-essential salt swig py27-pip unzip py27-mysqldb libsodium mysql-client patchelf
 pkgin -y rm salt py27-zmq
 
-pip install --egg esky bbfreeze
+pip install --no-use-wheel --egg esky bbfreeze
 
-cd $HERE
+## bzfreeze-loader
+COMPILE="gcc -fno-strict-aliasing -O2 -pipe -O2 -DHAVE_DB_185_H -I/usr/include -I/opt/local/include -I/opt/local/include/db4 -I/opt/local/include/gettext -I/opt/local/include/ncurses -DNDEBUG -O2 -pipe -O2 -DHAVE_DB_185_H -I/usr/include -I/opt/local/include -I/opt/local/include/db4 -I/opt/local/include/gettext -I/opt/local/include/ncurses -fPIC -I/opt/local/include/python2.7 -static-libgcc"
+LINK_OPTS="-L/opt/local/lib -L/opt/local/lib/python2.7/config -L/opt/local/lib -lsocket -lnsl -ldl -lrt -lm -static-libgcc"
+mkdir -p ${BLDPATH}
+cd ${BLDPATH}
 curl -kO 'https://pypi.python.org/packages/source/b/bbfreeze-loader/bbfreeze-loader-1.1.0.zip'
 unzip bbfreeze-loader-1.1.0.zip
+${COMPILE} -c bbfreeze-loader-1.1.0/_bbfreeze_loader/console.c -o ${BLDPATH}/console.o
+${COMPILE} -c bbfreeze-loader-1.1.0/_bbfreeze_loader/getpath.c -o ${BLDPATH}/getpath.o
+gcc ${BLDPATH}/console.o ${BLDPATH}/getpath.o /opt/local/lib/python2.7/config/libpython2.7.a ${LINK_OPTS} -o ${BLDPATH}/console.exe
+patchelf --set-rpath '$ORIGIN:$ORIGIN/../lib' ${BLDPATH}/console.exe
+find /opt/local -name console.exe -exec cp ${BLDPATH}/console.exe {} \;
 
-COMPILE="gcc -fno-strict-aliasing -O2 -pipe -O2 -DHAVE_DB_185_H -I/usr/include -I/opt/local/include -I/opt/local/include/db4 -I/opt/local/include/gettext -I/opt/local/include/ncurses -DNDEBUG -O2 -pipe -O2 -DHAVE_DB_185_H -I/usr/include -I/opt/local/include -I/opt/local/include/db4 -I/opt/local/include/gettext -I/opt/local/include/ncurses -fPIC -I/opt/local/include/python2.7 -static-libgcc"
-$COMPILE -c bbfreeze-loader-1.1.0/_bbfreeze_loader/console.c -o $HERE/console.o
-$COMPILE -c bbfreeze-loader-1.1.0/_bbfreeze_loader/getpath.c -o $HERE/getpath.o
-gcc $HERE/console.o $HERE/getpath.o /opt/local/lib/python2.7/config/libpython2.7.a -L/opt/local/lib -L/opt/local/lib/python2.7/config -L/opt/local/lib -lsocket -lnsl -ldl -lrt -lm -static-libgcc -o $HERE/console.exe
-patchelf --set-rpath '$ORIGIN:$ORIGIN/../lib' $HERE/console.exe
+## clone saltstack repo
+cd ${SALTBASE}
+git clone git://github.com/saltstack/salt -b 2016.3
 
-find /opt/local -name console.exe -exec mv $HERE/console.exe {} \;
+## salt requirements
+cd ${SALTBASE}/salt
+until pip install --no-use-wheel --egg -r pkg/smartos/esky/zeromq_requirements.txt ; do sleep 1 ; done ;
+until pip install --no-use-wheel --egg -r pkg/smartos/esky/raet_requirements.txt ; do sleep 1 ; done ;
 
-git clone git://github.com/saltstack/salt -b 2014.7
-cd $HERE/salt
-
-# install all requirements
-# (installing them as eggs seems to trigger esky pulling in the whole egg)
-# this step is buggy... I had to run them repeatedly until they succeeded...
-until pip install --egg -r pkg/smartos/esky/zeromq_requirements.txt ; do sleep 1 ; done ;
-until pip install --egg -r pkg/smartos/esky/raet_requirements.txt ; do sleep 1 ; done ;
-
-# install the sodium_grabber library
+## sodium grabber
+cd ${SALTBASE}/salt
 python2.7 pkg/smartos/esky/sodium_grabber_installer.py install
 
-# ugly workaround for odd zeromq linking breakage
-cp /opt/local/lib/libzmq.so.4 /opt/local/lib/python2.7/site-packages/pyzmq-13.1.0-py2.7-solaris-2.11-i86pc.64bit.egg/zmq/
-patchelf --set-rpath '$ORIGIN:$ORIGIN/../lib' /opt/local/lib/python2.7/site-packages/pyzmq-13.1.0-py2.7-solaris-2.11-i86pc.64bit.egg/zmq/libzmq.so.4
-cp /opt/local/lib/libsodium.so.13 /opt/local/lib/python2.7/site-packages/pyzmq-13.1.0-py2.7-solaris-2.11-i86pc.64bit.egg/zmq/
-patchelf --set-rpath '$ORIGIN:$ORIGIN/../lib' /opt/local/lib/python2.7/site-packages/pyzmq-13.1.0-py2.7-solaris-2.11-i86pc.64bit.egg/zmq/libsodium.so.13
+## cleanup
+rm -r ${BLDPATH}
 
-# at this point you have a build environment that you could set aside and reuse to run further builds.
-
-bash pkg/smartos/esky/build-tarball.sh
-
-# Upload packages into Manta
-#mmkdir -p /$MANTA_USER/public/salt
-#for file in dist/salt*; do mput -m /$MANTA_USER/public/salt -f $file; done;
+## build esky package
+cd ${SALTBASE}/salt
+pkg/smartos/esky/build-tarball.sh
 ```

pkg/smartos/esky/build-tarball.sh

@@ -24,5 +24,9 @@ chmod +x $BUILD_DIR/install/install.sh
 unzip -d $BUILD_DIR/bin dist/*.zip
 cp $BUILD_DIR/bin/*/libgcc_s.so.1 $BUILD_DIR/bin/
 find build/output/salt/bin/ -mindepth 1 -maxdepth 1 -type d -not -name appdata -exec mv {} $BUILD_DIR/bin/appdata/ \;
+PYZMQ=$(find ${BUILD_DIR}/bin/ -mindepth 1 -type d -name 'pyzmq-*.egg')/zmq
+find /opt/local/lib/ -maxdepth 1 -type l -regextype sed -regex '.*/libzmq.so.[0-9]\+$' -exec cp {} ${PYZMQ}/ \;
+find /opt/local/lib/ -maxdepth 1 -type l -regextype sed -regex '.*/libsodium.so.[0-9]\+$' -exec cp {} ${PYZMQ}/ \;
+find ${PYZMQ}/ -maxdepth 1 -type f -name '*.so.*' -exec patchelf --set-rpath '$ORIGIN:$ORIGIN/../../:$ORIGIN/../lib' {} \;
 gtar -C $BUILD_DIR/.. -czvf dist/salt-$(awk '/^Version:/{print $2}' < PKG-INFO)-esky-smartos.tar.gz salt
 echo "tarball built"

pkg/smartos/esky/zeromq_requirements.txt

@@ -2,5 +2,5 @@
 #-r ../../../requirements/zeromq.txt
 -r ../../../requirements/base.txt
 pycrypto>=2.6.1
-pyzmq == 13.1.0
+pyzmq
 -r requirements.txt