# -*- coding: utf-8 -*- # Maintainer: Erik Johnson (https://github.com/terminalmage) # # WARNING: This script will recursively remove the build and artifact # directories. # # This script is designed for speed, therefore it does not use mock and does not # run tests. It *will* install the build deps on the machine running the script. # # pylint: disable=file-perms,resource-leakage from __future__ import absolute_import, print_function import errno import glob import logging import os import re import shutil import subprocess import sys from optparse import OptionParser, OptionGroup logging.QUIET = 0 logging.GARBAGE = 1 logging.TRACE = 5 logging.addLevelName(logging.QUIET, 'QUIET') logging.addLevelName(logging.TRACE, 'TRACE') logging.addLevelName(logging.GARBAGE, 'GARBAGE') LOG_LEVELS = { 'all': logging.NOTSET, 'debug': logging.DEBUG, 'error': logging.ERROR, 'critical': logging.CRITICAL, 'garbage': logging.GARBAGE, 'info': logging.INFO, 'quiet': logging.QUIET, 'trace': logging.TRACE, 'warning': logging.WARNING, } log = logging.getLogger(__name__) # FUNCTIONS def _abort(msgs): ''' Unrecoverable error, pull the plug ''' if not isinstance(msgs, list): msgs = [msgs] for msg in msgs: log.error(msg) sys.stderr.write(msg + '\n\n') sys.stderr.write('Build failed. See log file for further details.\n') sys.exit(1) # HELPER FUNCTIONS def _init(): ''' Parse CLI options. ''' parser = OptionParser() parser.add_option('--platform', dest='platform', help='Platform (\'os\' grain)') parser.add_option('--log-level', dest='log_level', default='warning', help='Control verbosity of logging. Default: %default') # All arguments dealing with file paths (except for platform-specific ones # like those for SPEC files) should be placed in this group so that # relative paths are properly expanded. path_group = OptionGroup(parser, 'File/Directory Options') path_group.add_option('--source-dir', default='/testing', help='Source directory. Must be a git checkout. ' '(default: %default)') path_group.add_option('--build-dir', default='/tmp/salt-buildpackage', help='Build root, will be removed if it exists ' 'prior to running script. (default: %default)') path_group.add_option('--artifact-dir', default='/tmp/salt-packages', help='Location where build artifacts should be ' 'placed for Jenkins to retrieve them ' '(default: %default)') parser.add_option_group(path_group) # This group should also consist of nothing but file paths, which will be # normalized below. rpm_group = OptionGroup(parser, 'RPM-specific File/Directory Options') rpm_group.add_option('--spec', dest='spec_file', default='/tmp/salt.spec', help='Spec file to use as a template to build RPM. ' '(default: %default)') parser.add_option_group(rpm_group) opts = parser.parse_args()[0] # Expand any relative paths for group in (path_group, rpm_group): for path_opt in [opt.dest for opt in group.option_list]: path = getattr(opts, path_opt) if not os.path.isabs(path): # Expand ~ or ~user path = os.path.expanduser(path) if not os.path.isabs(path): # Still not absolute, resolve '..' path = os.path.realpath(path) # Update attribute with absolute path setattr(opts, path_opt, path) # Sanity checks problems = [] if not opts.platform: problems.append('Platform (\'os\' grain) required') if not os.path.isdir(opts.source_dir): problems.append('Source directory {0} not found' .format(opts.source_dir)) try: shutil.rmtree(opts.build_dir) except OSError as exc: if exc.errno not in (errno.ENOENT, errno.ENOTDIR): problems.append('Unable to remove pre-existing destination ' 'directory {0}: {1}'.format(opts.build_dir, exc)) finally: try: os.makedirs(opts.build_dir) except OSError as exc: problems.append('Unable to create destination directory {0}: {1}' .format(opts.build_dir, exc)) try: shutil.rmtree(opts.artifact_dir) except OSError as exc: if exc.errno not in (errno.ENOENT, errno.ENOTDIR): problems.append('Unable to remove pre-existing artifact directory ' '{0}: {1}'.format(opts.artifact_dir, exc)) finally: try: os.makedirs(opts.artifact_dir) except OSError as exc: problems.append('Unable to create artifact directory {0}: {1}' .format(opts.artifact_dir, exc)) # Create log file in the artifact dir so it is sent back to master if the # job fails opts.log_file = os.path.join(opts.artifact_dir, 'salt-buildpackage.log') if problems: _abort(problems) return opts def _move(src, dst): ''' Wrapper around shutil.move() ''' try: os.remove(os.path.join(dst, os.path.basename(src))) except OSError as exc: if exc.errno != errno.ENOENT: _abort(exc) try: shutil.move(src, dst) except shutil.Error as exc: _abort(exc) def _run_command(args): log.info('Running command: {0}'.format(args)) proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate() if stdout: log.debug('Command output: \n{0}'.format(stdout)) if stderr: log.error(stderr) log.info('Return code: {0}'.format(proc.returncode)) return stdout, stderr, proc.returncode def _make_sdist(opts, python_bin='python'): os.chdir(opts.source_dir) stdout, stderr, rcode = _run_command([python_bin, 'setup.py', 'sdist']) if rcode == 0: # Find the sdist with the most recently-modified metadata sdist_path = max( glob.iglob(os.path.join(opts.source_dir, 'dist', 'salt-*.tar.gz')), key=os.path.getctime ) log.info('sdist is located at {0}'.format(sdist_path)) return sdist_path else: _abort('Failed to create sdist') # BUILDER FUNCTIONS def build_centos(opts): ''' Build an RPM ''' log.info('Building CentOS RPM') log.info('Detecting major release') try: with open('/etc/redhat-release', 'r') as fp_: redhat_release = fp_.read().strip() major_release = int(redhat_release.split()[2].split('.')[0]) except (ValueError, IndexError): _abort('Unable to determine major release from /etc/redhat-release ' 'contents: \'{0}\''.format(redhat_release)) except IOError as exc: _abort('{0}'.format(exc)) log.info('major_release: {0}'.format(major_release)) define_opts = [ '--define', '_topdir {0}'.format(os.path.join(opts.build_dir)) ] build_reqs = ['rpm-build'] if major_release == 5: python_bin = 'python26' define_opts.extend(['--define', 'dist .el5']) if os.path.exists('/etc/yum.repos.d/saltstack.repo'): build_reqs.extend(['--enablerepo=saltstack']) build_reqs.extend(['python26-devel']) elif major_release == 6: build_reqs.extend(['python-devel']) elif major_release == 7: build_reqs.extend(['python-devel', 'systemd-units']) else: _abort('Unsupported major release: {0}'.format(major_release)) # Install build deps _run_command(['yum', '-y', 'install'] + build_reqs) # Make the sdist try: sdist = _make_sdist(opts, python_bin=python_bin) except NameError: sdist = _make_sdist(opts) # Example tarball names: # - Git checkout: salt-2014.7.0rc1-1584-g666602e.tar.gz # - Tagged release: salt-2014.7.0.tar.gz tarball_re = re.compile(r'^salt-([^-]+)(?:-(\d+)-(g[0-9a-f]+))?\.tar\.gz$') try: base, offset, oid = tarball_re.match(os.path.basename(sdist)).groups() except AttributeError: _abort('Unable to extract version info from sdist filename \'{0}\'' .format(sdist)) if offset is None: salt_pkgver = salt_srcver = base else: salt_pkgver = '.'.join((base, offset, oid)) salt_srcver = '-'.join((base, offset, oid)) log.info('salt_pkgver: {0}'.format(salt_pkgver)) log.info('salt_srcver: {0}'.format(salt_srcver)) # Setup build environment for build_dir in 'BUILD BUILDROOT RPMS SOURCES SPECS SRPMS'.split(): path = os.path.join(opts.build_dir, build_dir) try: os.makedirs(path) except OSError: pass if not os.path.isdir(path): _abort('Unable to make directory: {0}'.format(path)) # Get sources into place build_sources_path = os.path.join(opts.build_dir, 'SOURCES') rpm_sources_path = os.path.join(opts.source_dir, 'pkg', 'rpm') _move(sdist, build_sources_path) for src in ('salt-master', 'salt-syndic', 'salt-minion', 'salt-api', 'salt-master.service', 'salt-syndic.service', 'salt-minion.service', 'salt-api.service', 'README.fedora', 'logrotate.salt', 'salt.bash'): shutil.copy(os.path.join(rpm_sources_path, src), build_sources_path) # Prepare SPEC file spec_path = os.path.join(opts.build_dir, 'SPECS', 'salt.spec') with open(opts.spec_file, 'r') as spec: spec_lines = spec.read().splitlines() with open(spec_path, 'w') as fp_: for line in spec_lines: if line.startswith('%global srcver '): line = '%global srcver {0}'.format(salt_srcver) elif line.startswith('Version: '): line = 'Version: {0}'.format(salt_pkgver) fp_.write(line + '\n') # Do the thing cmd = ['rpmbuild', '-ba'] cmd.extend(define_opts) cmd.append(spec_path) stdout, stderr, rcode = _run_command(cmd) if rcode != 0: _abort('Build failed.') packages = glob.glob( os.path.join( opts.build_dir, 'RPMS', 'noarch', 'salt-*{0}*.noarch.rpm'.format(salt_pkgver) ) ) packages.extend( glob.glob( os.path.join( opts.build_dir, 'SRPMS', 'salt-{0}*.src.rpm'.format(salt_pkgver) ) ) ) return packages # MAIN if __name__ == '__main__': opts = _init() print('Starting {0} build. Progress will be logged to {1}.' .format(opts.platform, opts.log_file)) # Setup logging log_format = '%(asctime)s.%(msecs)03d %(levelname)s: %(message)s' log_datefmt = '%H:%M:%S' log_level = LOG_LEVELS[opts.log_level] \ if opts.log_level in LOG_LEVELS \ else LOG_LEVELS['warning'] logging.basicConfig(filename=opts.log_file, format=log_format, datefmt=log_datefmt, level=LOG_LEVELS[opts.log_level]) if opts.log_level not in LOG_LEVELS: log.error('Invalid log level \'{0}\', falling back to \'warning\'' .format(opts.log_level)) # Build for the specified platform if not opts.platform: _abort('Platform required') elif opts.platform.lower() == 'centos': artifacts = build_centos(opts) else: _abort('Unsupported platform \'{0}\''.format(opts.platform)) msg = ('Build complete. Artifacts will be stored in {0}' .format(opts.artifact_dir)) log.info(msg) print(msg) # pylint: disable=C0325 for artifact in artifacts: shutil.copy(artifact, opts.artifact_dir) log.info('Copied {0} to artifact directory'.format(artifact)) log.info('Done!')