MSPF-66: Title for github

* Jenkinsfile Targets have been updated. * Makefile Update to use build utils and tag container with date on push. Testing the shared repo cache. * bootstrap ** packer.sh Use busybox to bootstrap from scratch; ** packer.json.template Use busybox to bootstrap from scratch; <SHARED> directory placeholder; * shared Deleted.
2024-11-06 18:25:17 +00:00 · 2016-08-03 16:58:16 +03:00 · 2016-08-03 16:58:16 +03:00 · 868a4e4508
commit 868a4e4508
parent 059d90ebf2
11 changed files with 272 additions and 57 deletions
--- a/9
+++ b/9
@ -23,13 +23,6 @@ def images_pipeline(String repoName, String agentLabel, String artiFacts, Closur
    } catch (Exception e) {
      slackSend color: 'danger', message: "<${env.BUILD_URL}|Build ${env.BUILD_NUMBER}> for ${env.REPO_NAME} by ${env.COMMIT_AUTHOR} has failed on branch ${env.BRANCH_NAME} at stage: ${env.STAGE_NAME} (jenkins node: ${env.NODE_NAME})."
      throw e; // rethrow so the build is considered failed
-    } finally {
-      storeCtLog()
-
-      // runStage('store artifacts') {
-      //   storeArtifacts(artiFacts)
-      // }
-
    }
  }
 }
@ -42,7 +35,7 @@ images_pipeline("images", 'docker-host', "_build/") {
  }
  if (env.BRANCH_NAME == 'master') {
    runStage('docker image push') {
-      sh 'CONTAINER=bootstrap make push'
+      sh 'make push'
    }
  }
 }
--- a/45
+++ b/45
@ -1,48 +1,23 @@
-PACKER := $(shell which packer 2>/dev/null || which ./packer)
-PCONF  := packer.json
-PBUILD := $(PACKER) build $(PCONF)
+include build-utils/utils-image.mk
+include build-utils/utils-repo.mk
+PACKER := $(shell which packer 2>/dev/null || echo ./packer)
 BASE_DIR := $(shell pwd)

 DOCKER := $(shell which docker 2>/dev/null)
-DREPO  := dr.rbkmoney.com/rbkmoney
-CONTAINER ?=
-
-BAKKA_SU_PRIVKEY ?=
-BAKKA_SU_URI_PREFIX := $(if $(BAKKA_SU_PRIVKEY),git+ssh,git)://git.bakka.su
-BAKKA_SU_SSH_COMMAND := $(shell which ssh) -o User=git -o StrictHostKeyChecking=no $(if $(BAKKA_SU_PRIVKEY),-i $(BAKKA_SU_PRIVKEY),)

+PULL := true

 .PHONY: bootstrap push

-# portage
-shared/portage/.git/config:
-	rm -rf "$(BASE_DIR)/shared/portage" \
-	&& GIT_SSH_COMMAND="$(BAKKA_SU_SSH_COMMAND)" git clone \
-	"$(BAKKA_SU_URI_PREFIX)/gentoo-mirror" --depth 1 \
-	"$(BASE_DIR)/shared/portage"
-
-# overlays
-shared/baka-bakka/.git/config:
-	rm -rf "$(BASE_DIR)/shared/baka-bakka" \
-	&& GIT_SSH_COMMAND="$(BAKKA_SU_SSH_COMMAND)" git clone \
-	"$(BAKKA_SU_URI_PREFIX)/baka-bakka" --depth 1 \
-	"$(BASE_DIR)/shared/baka-bakka"
-
 # bootstrap
 bootstrap: bootstrap/.state

-bootstrap/.state: $(PACKER) shared/portage/.git/config bootstrap/packer.json bootstrap/packer.sh bootstrap/portage.make.conf
-	cd $(BASE_DIR)/$(dir $@) && $(PBUILD) && touch .state
+bootstrap/.state: $(PACKER) $(IMAGES_SHARED)/portage/.git bootstrap/packer.json bootstrap/packer.sh bootstrap/portage.make.conf
+	cd $(BASE_DIR)/$(dir $@) && $(PACKER) build packer.json && touch .state

 bootstrap/packer.json: bootstrap/packer.json.template
-	sed 's:<PATH>:$(BASE_DIR):g' $< > $@
+	sed -e 's:<PATH>:$(BASE_DIR):g' -e 's:<SHARED>:$(IMAGES_SHARED):g' $< > $@

-
-# docker push
-# make sure to run `docker login` before
-push: $(CONTAINER)/.state $(DOCKER) ~/.docker/config.json
-	$(DOCKER) push $(DREPO)/$(CONTAINER)
-
-
-~/.docker/config.json:
-	test -f ~/.docker/config.json || (echo "Please run: docker login" ; exit 1)
+push:
+	$(DOCKER) tag "$(DREPO)/bootstrap" "$(DREPO)/bootstrap:$(shell date --rfc-3339=date)"
+	$(DOCKER) push "$(DREPO)/bootstrap:$(shell date --rfc-3339=date)"
--- a/bootstrap/packer.json.template
+++ b/bootstrap/packer.json.template
@ -2,10 +2,11 @@
    "builders": [
        {
            "type": "docker",
-            "image": "gentoo/stage3-amd64-hardened",
+            "image": "busybox",
+	    "run_command": ["-d", "-i", "-t", "{{.Image}}", "/bin/sh"],
            "volumes": {
                "<PATH>/bootstrap/": "/tmp/data",
-                "<PATH>/shared/portage": "/usr/portage"
+                "<SHARED>/portage": "/usr/portage"
            },
            "commit": "true"
        }
--- a/bootstrap/packer.sh
+++ b/bootstrap/packer.sh
@ -1,8 +1,72 @@
-#! /bin/bash
+#!/bin/sh
+# usage: stage-arch arch-name suffix

+VerifyHashOfStage3() {
+    # First param is package tarball, 2nd is the *.DIGEST file
+    test_sum=$(awk -v myvar="$1" '$2==myvar {for(i=1; i<=1; i++) { print $1; exit}}' "${2}")
+    calculated_sum=$(sha512sum "${1}" | awk '{print $1}' -)
+    if [[ "$test_sum" == "$calculated_sum" ]]; then
+	return 0
+    else
+	return 1
+    fi
+}
+
+suffix="-hardened+nomultilib" # e.g. -hardened
+arch="amd64"
+dist="http://gentoo.bakka.su/releases/${arch}/autobuilds/"
+
+echo "-I- dist: ${dist}"
+echo "-I- arch: ${arch} suffix: ${suffix}"
+
+echo '-=- Preparing the working directory'
+mkdir newWorldOrder; cd newWorldOrder || exit 1
+cp /bin/busybox . || exit 1
+echo '-ok'
+
+echo "-=- Downloading ${dist}/latest-stage3-${arch}${suffix}.txt"
+wget -q "${dist}/latest-stage3-${arch}${suffix}.txt" || exit 1
+echo '-ok'
+stage3path="$(cat latest-stage3-${arch}${suffix}.txt | tail -n 1 | cut -f 1 -d ' ')"
+stage3="$(basename ${stage3path})"
+echo "-I- latest stage3: ${stage3path}"
+
+echo "-=- Downloading ${dist}/${stage3path} and its DIGESTS"
+wget -q -c "${dist}/${stage3path}" "${dist}/${stage3path}.DIGESTS" || exit 1
+echo "-ok"
+if VerifyHashOfStage3 "${stage3}" "${stage3}.DIGESTS"; then
+    echo "-ok DIGEST verification passed, sha512 hashes match."
+else
+    echo "-!! DIGEST verification failed!"
+    exit 1
+fi
+echo "-=- Unpacking ${stage3}"
+bunzip2 -c "${stage3}" | tar --exclude "./etc/hosts" --exclude "./sys/*" -xf - || exit 1
+echo "-ok"
+echo "-=- Removing ${stage3}"
+/newWorldOrder/busybox rm -f "${stage3}" || exit 1
+echo "-ok"
+
+echo "-=- Installing unpacked contents"
+ls /usr -la
+/newWorldOrder/busybox rm -rf /lib* /usr/sbin /var /bin /sbin /opt /mnt /media /root /home /run || exit 1
+/newWorldOrder/busybox cp -fRap lib* bin boot home media mnt opt root run sbin tmp usr var / || exit 1
+/newWorldOrder/busybox cp -fRap etc/* /etc/ || exit 1
+echo "-ok"
+
+echo "-=- Cleaning up"
+cd /
+/newWorldOrder/busybox rm -rf /newWorldOrder /build.sh /linuxrc || exit 1
+echo "-ok"
+
+echo "-I- Bootstrapped ${stage3path} into /"
+echo "-I- Here begins the New World Order"
+
+export EMERGE="emerge -q"
+
+/bin/bash <<EOL
 source /lib/gentoo/functions.sh

-EMERGE="emerge -q"

 ebegin "Setting locales to generate"
 cat <<EOF> /etc/locale.gen
@ -89,3 +153,4 @@ if [ ! -d /var/salt ]; then
    mkdir -p /var/salt
    eend $? || exit $?
 fi
+EOL
--- a/build-utils/functions.sh
+++ b/build-utils/functions.sh
@ -0,0 +1,117 @@
+# Copyright (c) 2007-2009 Roy Marples <roy@marples.name>
+# Released under the 2-clause BSD license.
+
+# Allow any sh script to work with einfo functions and friends
+# We also provide a few helpful functions for other programs to use
+
+RC_GOT_FUNCTIONS="yes"
+
+eindent()
+{
+	: $(( EINFO_INDENT = ${EINFO_INDENT:-0} + 2 ))
+	[ "$EINFO_INDENT" -gt 40 ] && EINFO_INDENT=40
+	export EINFO_INDENT
+}
+
+eoutdent()
+{
+	: $(( EINFO_INDENT = ${EINFO_INDENT:-0} - 2 ))
+	[ "$EINFO_INDENT" -lt 0 ] && EINFO_INDENT=0
+	return 0
+}
+
+yesno()
+{
+	[ -z "$1" ] && return 1
+
+	# Check the value directly so people can do:
+	# yesno ${VAR}
+	case "$1" in
+		[Yy][Ee][Ss]|[Tt][Rr][Uu][Ee]|[Oo][Nn]|1) return 0;;
+		[Nn][Oo]|[Ff][Aa][Ll][Ss][Ee]|[Oo][Ff][Ff]|0) return 1;;
+	esac
+
+	# Check the value of the var so people can do:
+	# yesno VAR
+	# Note: this breaks when the var contains a double quote.
+	local value=
+	eval value=\"\$$1\"
+	case "$value" in
+		[Yy][Ee][Ss]|[Tt][Rr][Uu][Ee]|[Oo][Nn]|1) return 0;;
+		[Nn][Oo]|[Ff][Aa][Ll][Ss][Ee]|[Oo][Ff][Ff]|0) return 1;;
+		*) vewarn "\$$1 is not set properly"; return 1;;
+	esac
+}
+
+rc_runlevel()
+{
+    rc-status --runlevel
+}
+
+_sanitize_path()
+{
+	local IFS=":" p= path=
+	for p in $PATH; do
+		case "$p" in
+			/lib64/rc/bin|/lib64/rc/sbin);;
+			/bin|/sbin|/usr/bin|/usr/sbin);;
+			/usr/bin|/usr/sbin);;
+			/usr/local/bin|/usr/local/sbin);;
+			*) path="$path${path:+:}$p";;
+		esac
+	done
+	echo "$path"
+}
+
+# Allow our scripts to support zsh
+if [ -n "$ZSH_VERSION" ]; then
+	emulate sh
+	NULLCMD=:
+	alias -g '${1+"$@"}'='"$@"'
+	setopt NO_GLOB_SUBST
+fi
+
+# Make a sane PATH
+_PREFIX=
+_PKG_PREFIX=/usr
+_LOCAL_PREFIX=/usr/local
+_LOCAL_PREFIX=${_LOCAL_PREFIX:-/usr/local}
+_PATH=/lib64/rc/bin
+case "$_PREFIX" in
+	"$_PKG_PREFIX"|"$_LOCAL_PREFIX") ;;
+	*) _PATH="$_PATH:$_PREFIX/bin:$_PREFIX/sbin";;
+esac
+_PATH="$_PATH":/bin:/sbin:/usr/bin:/usr/sbin
+
+if [ -n "$_PKG_PREFIX" ]; then
+	_PATH="$_PATH:$_PKG_PREFIX/bin:$_PKG_PREFIX/sbin"
+fi
+if [ -n "$_LOCAL_PREFIX" ]; then
+	_PATH="$_PATH:$_LOCAL_PREFIX/bin:$_LOCAL_PREFIX/sbin"
+fi
+_path="$(_sanitize_path "$PATH")"
+PATH="$_PATH${_path:+:}$_path" ; export PATH
+unset _sanitize_path _PREFIX _PKG_PREFIX _LOCAL_PREFIX _PATH _path
+
+for arg; do
+	case "$arg" in
+		--nocolor|--nocolour|-C)
+			EINFO_COLOR="NO" ; export EINFO_COLOR
+			;;
+	esac
+done
+
+if [ -t 1 ] && yesno "${EINFO_COLOR:-YES}"; then
+	if [ -z "$GOOD" ]; then
+		eval $(eval_ecolors)
+	fi
+else
+	# We need to have shell stub functions so our init scripts can remember
+	# the last ecmd
+	for _e in ebegin eend error errorn einfo einfon ewarn ewarnn ewend \
+		vebegin veend veinfo vewarn vewend; do
+		eval "$_e() { local _r; command $_e \"\$@\"; _r=\$?; \
+		EINFO_LASTCMD=$_e; export EINFO_LASTCMD ; return \$_r; }"
+	done
+	unset _e
+fi
--- a/build-utils/repo_init.sh
+++ b/build-utils/repo_init.sh
@ -0,0 +1,29 @@
+#!/bin/bash
+source build-utils/functions.sh
+
+d_repo="$1"
+remote_uri="$2"
+
+if [[ "${remote_uri}" == "git+ssh"* ]]; then
+    export GIT_SSH_COMMAND="$(which ssh) -o StrictHostKeyChecking=no -o User=git $([ -n "${SSH_PRIVKEY}" ] && echo -i "${SSH_PRIVKEY}")"
+    einfo "GIT_SSH_COMMAND: ${GIT_SSH_COMMAND}"
+fi
+if [[ -d "${d_repo}/.git" ]]; then
+    einfo "Syncing repository ${d_repo}"
+    git -C "${d_repo}" pull --depth 1 || exit $?
+else
+    einfo "Initialising repository ${d_repo}"
+    eindent
+    if [[ -d "${d_repo}" ]]; then
+	ebegin "Removing directory ${d_repo}"
+	rmdir "${d_repo}"
+	eend $? "Failed to remove ${d_repo}, do that yourself" || exit $?
+    fi
+    ebegin "Creating directory ${d_repo}"
+    mkdir -p "${d_repo}"
+    eend $? "Failed to create directory ${d_repo}" || exit $?
+    einfo "Clonning salt states repository ${d_repo}"
+    git clone --depth 1 "${remote_uri}" "${d_repo}" || exit $?
+    eoutdent
+fi
+
--- a/build-utils/utils-image.mk
+++ b/build-utils/utils-image.mk
@ -0,0 +1,12 @@
+DOCKER := $(shell which docker 2>/dev/null)
+DREPO  := dr.rbkmoney.com/rbkmoney
+
+DOCKER_SQUASH := $(shell which docker-squash 2>/dev/null || echo ~/.local/bin/docker-squash)
+
+IMAGE_NAME ?=
+SRC_TAG ?=
+PUSH_TAG ?=
+
+push:
+	$(DOCKER) tag "$(IMAGE_NAME):$(SRC_TAG)" "$(IMAGE_NAME):$(PUSH_TAG)"
+	$(DOCKER) push "$(IMAGE_NAME):$(PUSH_TAG)"
--- a/build-utils/utils-repo.mk
+++ b/build-utils/utils-repo.mk
@ -0,0 +1,32 @@
+REPO_INIT := build-utils/repo_init.sh
+IMAGES_SHARED := $(shell echo "${HOME}")/.cache/rbkmoney/images/shared
+GITHUB_PRIVKEY ?=
+GITHUB_URI_PREFIX := git+ssh://github.com
+GITHUB_SSH_COMMAND := $(shell which ssh) -o User=git -o StrictHostKeyChecking=no $(if $(GITHUB_PRIVKEY),-i $(GITHUB_PRIVKEY),)
+
+BAKKA_SU_PRIVKEY ?=
+BAKKA_SU_URI_PREFIX := $(if $(BAKKA_SU_PRIVKEY),git+ssh,git)://git.bakka.su
+BAKKA_SU_SSH_COMMAND := $(shell which ssh) -o User=git -o StrictHostKeyChecking=no $(if $(BAKKA_SU_PRIVKEY),-i $(BAKKA_SU_PRIVKEY),)
+
+# portage
+$(IMAGES_SHARED)/portage/.git: .git
+	$(if $(BAKKA_SU_PRIVKEY),SSH_PRIVKEY="$(BAKKA_SU_PRIVKEY)",) "$(REPO_INIT)" \
+	"$(IMAGES_SHARED)/portage" "$(BAKKA_SU_URI_PREFIX)/gentoo-mirror"
+
+# overlays
+$(IMAGES_SHARED)/overlays/rbkmoney/.git: .git
+	$(if $(GITHUB_PRIVKEY),SSH_PRIVKEY="$(GITHUB_PRIVKEY)",) "$(REPO_INIT)" \
+	"$(IMAGES_SHARED)/overlays/rbkmoney" "$(GITHUB_URI_PREFIX)/rbkmoney/gentoo-overlay"
+
+$(IMAGES_SHARED)/overlays/baka-bakka/.git: .git
+	$(if $(BAKKA_SU_PRIVKEY),SSH_PRIVKEY="$(BAKKA_SU_PRIVKEY)",) "$(REPO_INIT)" \
+	"$(IMAGES_SHARED)/overlays/baka-bakka" "$(BAKKA_SU_URI_PREFIX)/baka-bakka"
+
+# salt
+$(IMAGES_SHARED)/salt/rbkmoney/.git: .git
+	$(if $(GITHUB_PRIVKEY),SSH_PRIVKEY="$(GITHUB_PRIVKEY)",) "$(REPO_INIT)" \
+	"$(IMAGES_SHARED)/salt/rbkmoney" "$(GITHUB_URI_PREFIX)/rbkmoney/salt-main"
+
+$(IMAGES_SHARED)/salt/common/.git: .git
+	$(if $(BAKKA_SU_PRIVKEY),SSH_PRIVKEY="$(BAKKA_SU_PRIVKEY)",) "$(REPO_INIT)" \
+	"$(IMAGES_SHARED)/salt/common" "$(BAKKA_SU_URI_PREFIX)/salt-common"
--- a/shared/baka-bakka
+++ b/shared/baka-bakka
@ -1 +0,0 @@
-Subproject commit 317886fc086385e50dbd14c36b2a42475c5b735e
--- a/shared/id_ed25519-rbkmoney-all
+++ b/shared/id_ed25519-rbkmoney-all
@ -1,7 +0,0 @@
-----BEGIN OPENSSH PRIVATE KEY-----
-b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
-QyNTUxOQAAACDUdGE974bgpHITGBuGGRFBADxh0JaKwWR4je7Z0fyfzQAAAJBWg49jVoOP
-YwAAAAtzc2gtZWQyNTUxOQAAACDUdGE974bgpHITGBuGGRFBADxh0JaKwWR4je7Z0fyfzQ
-AAAED56BBIUui9IjfdyNj0tGtr8W1Ie16mCYwekvZsjXbhCtR0YT3vhuCkchMYG4YZEUEA
-PGHQlorBZHiN7tnR/J/NAAAACWN5YUB0eXBlNQECAwQ=
-----END OPENSSH PRIVATE KEY-----
--- a/shared/portage
+++ b/shared/portage
@ -1 +0,0 @@
-Subproject commit 35a9b56d7825743a7dc5158b1922c9bde131f2ce
				`@ -1 +0,0 @@`
				`Subproject commit 317886fc086385e50dbd14c36b2a42475c5b735e`
				`@ -1 +0,0 @@`
				`Subproject commit 35a9b56d7825743a7dc5158b1922c9bde131f2ce`