first commit

.gitignore

@@ -0,0 +1,12 @@
+.vagrant
+packer.json
+.squashed
+.state
+shared/portage/**
+shared/baka-bakka/**
+shared/rbkmoney/**
+shared/salt/**
+*~
+*.sublime-workspace
+.DS_Store
+

Jenkinsfile

@@ -0,0 +1,48 @@
+#!groovy
+// -*- mode: groovy -*-
+
+// Args:
+// GitHub repo name
+// Jenkins agent label
+// Tracing artifacts to be stored alongside build logs
+def images_pipeline(String repoName, String agentLabel, String artiFacts, Closure body) {
+  node(agentLabel) {
+    try {
+      env.REPO_NAME = repoName
+      runStage('git checkout') {
+	checkout scm
+	//sh 'git submodule update --init'
+	sh 'git --no-pager log -1 --pretty=format:"%an" > .commit_author'
+	env.COMMIT_AUTHOR = readFile('.commit_author').trim()
+      }
+      wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) {
+	body.call()
+      }
+
+      slackSend color: 'good', message: "<${env.BUILD_URL}|Build ${env.BUILD_NUMBER}> for ${env.REPO_NAME} by ${env.COMMIT_AUTHOR} has passed on branch ${env.BRANCH_NAME} (jenkins node: ${env.NODE_NAME})."
+    } catch (Exception e) {
+      slackSend color: 'danger', message: "<${env.BUILD_URL}|Build ${env.BUILD_NUMBER}> for ${env.REPO_NAME} by ${env.COMMIT_AUTHOR} has failed on branch ${env.BRANCH_NAME} at stage: ${env.STAGE_NAME} (jenkins node: ${env.NODE_NAME})."
+      throw e; // rethrow so the build is considered failed
+    } finally {
+      storeCtLog()
+
+      // runStage('store artifacts') {
+      //   storeArtifacts(artiFacts)
+      // }
+
+    }
+  }
+}
+
+images_pipeline("images", 'docker-host', "_build/") {
+  withCredentials([[$class: 'FileBinding', credentialsId: 'bakka-su-rbkmoney-all', variable: 'BAKKA_SU_PRIVKEY']]) {
+    runStage('bootstrap image build') {
+      sh 'make bootstrap'
+    }
+  }
+  if (env.BRANCH_NAME == 'master') {
+    runStage('docker image push') {
+      sh 'CONTAINER=bootstrap make push'
+    }
+  }
+}

Makefile

@@ -0,0 +1,48 @@
+PACKER := $(shell which packer 2>/dev/null || which ./packer)
+PCONF  := packer.json
+PBUILD := $(PACKER) build $(PCONF)
+BASE_DIR := $(shell pwd)
+
+DOCKER := $(shell which docker 2>/dev/null)
+DREPO  := dr.rbkmoney.com/rbkmoney
+CONTAINER ?=
+
+BAKKA_SU_PRIVKEY ?=
+BAKKA_SU_URI_PREFIX := $(if $(BAKKA_SU_PRIVKEY),git+ssh,git)://git.bakka.su
+BAKKA_SU_SSH_COMMAND := $(shell which ssh) -o User=git -o StrictHostKeyChecking=no $(if $(BAKKA_SU_PRIVKEY),-i $(BAKKA_SU_PRIVKEY),)
+
+
+.PHONY: bootstrap push 
+
+# portage
+shared/portage/.git/config:
+	rm -rf "$(BASE_DIR)/shared/portage" \
+	&& GIT_SSH_COMMAND="$(BAKKA_SU_SSH_COMMAND)" git clone \
+	"$(BAKKA_SU_URI_PREFIX)/gentoo-mirror" --depth 1 \
+	"$(BASE_DIR)/shared/portage"
+
+# overlays
+shared/baka-bakka/.git/config:
+	rm -rf "$(BASE_DIR)/shared/baka-bakka" \
+	&& GIT_SSH_COMMAND="$(BAKKA_SU_SSH_COMMAND)" git clone \
+	"$(BAKKA_SU_URI_PREFIX)/baka-bakka" --depth 1 \
+	"$(BASE_DIR)/shared/baka-bakka"
+
+# bootstrap
+bootstrap: bootstrap/.state
+
+bootstrap/.state: $(PACKER) shared/portage/.git/config bootstrap/packer.json bootstrap/packer.sh bootstrap/portage.make.conf
+	cd $(BASE_DIR)/$(dir $@) && $(PBUILD) && touch .state
+
+bootstrap/packer.json: bootstrap/packer.json.template
+	sed 's:<PATH>:$(BASE_DIR):g' $< > $@
+
+
+# docker push
+# make sure to run `docker login` before
+push: $(CONTAINER)/.state $(DOCKER) ~/.docker/config.json
+	$(DOCKER) push $(DREPO)/$(CONTAINER)
+
+
+~/.docker/config.json:
+	test -f ~/.docker/config.json || (echo "Please run: docker login" ; exit 1)

README.md

@@ -0,0 +1,62 @@
+#Images
+
+Скрипты и инструменты для создания образов (vm images, docker containers, vagrant boxes, etc), используемых в платформе и инфраструктуре _rbkmoney_.
+
+## Prerequisities
+### Vagrant box
+На OS X платформе рекомендуется использовать _Vagrant rbkmoney dev box_
+
+```
+cd dev
+vagrant up
+vagrant ssh
+cd /base_images/
+```
+
+
+### Docker hub
+Перед созданием контейнеров рекомендуется авторизоваться в _docker hub_ и получить там доступ к __rbkmoney__ репозиториям.
+Большинство `make` команд подразумевает использование готовых _приватных_ контейнеров __rbkmoney__. Только _scratch builds_ возможны без авторизации.
+Для авторизации сессии в _docker hub_ надо выполнить команду `docker login`.
+
+## Containers hierarchy
+```
+bootstrap
+    service_base
+        service_erlang
+        service_go
+        service_java
+    host
+```
+
+## HowTo
+### Build a container
+Создать контейнер `<container>` на основе готового родительского контейнера, согласно _containers hierarchy_.
+Последняя версия родительского контейнера скачивается из репозитория _rbkmoney_ на _docker hub_ (если локальная версия соответствует последней из _docker hub_, _docker_ использует её после проверки).
+
+```
+make <container>
+```
+
+Смотри список доступных значений `<container>` в __Containers hierarchy__.
+
+### Build a container from scratch
+Создать контейнер `<container>` с нуля, т.е. построить контейнер и все родительские контейнеры в иерархии.
+Если какой-либо родительский контейнер уже строился локально, то он может быть взят из локального _docker image registry_
+(если его зависимости не изменялись с последнего билда - стандартная логика `make`). `docker pull` использован не будет.
+
+```
+FROM_SCRATCH=true make <container>
+```
+
+### Rebuild a container
+Если необходимо перестроить уже созданный контейнер, надо удалить файл `.state` в папке контейнера: `<container>/.state`.
+В противном случае `make` не запустит пересборку при отсутствии изменений в зависимостях контейнера.
+
+### Push a container
+Сохранить контейнер <container> с тегом `latest` в __rbkmoney__ _docker hub_.
+
+```
+CONTAINER=<container> make push
+```
+

bootstrap/packer.json.template

@@ -0,0 +1,26 @@
+{
+    "builders": [
+        {
+            "type": "docker",
+            "image": "gentoo/stage3-amd64-hardened",
+            "volumes": {
+                "<PATH>/bootstrap/": "/tmp/data",
+                "<PATH>/shared/portage": "/usr/portage"
+            },
+            "commit": "true"
+        }
+    ],
+    "provisioners": [
+        {
+            "type": "shell",
+            "script": "<PATH>/bootstrap/packer.sh"
+        }
+    ],
+    "post-processors": [
+        {
+            "type": "docker-tag",
+            "repository": "dr.rbkmoney.com/rbkmoney/bootstrap"
+        }
+    ]
+}
+

bootstrap/packer.sh

@@ -0,0 +1,91 @@
+#! /bin/bash
+
+source /lib/gentoo/functions.sh
+
+EMERGE="emerge -q"
+
+ebegin "Setting locales to generate"
+cat <<EOF> /etc/locale.gen
+en_DK.UTF-8 UTF-8
+EOF
+eend $? "Failed" || exit $?
+locale-gen || exit $?
+
+eselect locale set en_DK.utf8 || exit $?
+
+ebegin "Downloading CA for the package repository"
+mkdir -p /usr/local/share/ca-certificates \
+    && wget -q http://bakka.su/ca/baka_bakka.crt -O /usr/local/share/ca-certificates/baka_bakka.crt
+eend $? "Failed" || exit $?
+ebegin "Updating CA cerificates"
+update-ca-certificates --fresh > /dev/null
+eend $? "Failed" || exit $?
+
+ebegin "Copying portage/make.conf"
+cp /tmp/data/portage.make.conf /etc/portage/make.conf
+eend $? "Failed" || exit $?
+
+ebegin "Adding repos.conf/gentoo"
+mkdir -p /etc/portage/repos.conf \
+    && cat <<EOF> /etc/portage/repos.conf/gentoo.conf
+[DEFAULT]
+main-repo = gentoo
+
+[gentoo]
+location = /usr/portage
+sync-type = git
+sync-uri = git://git.bakka.su/gentoo-mirror
+EOF
+eend $? "Failed" || exit $?
+
+ebegin "Selecting profile"
+eselect profile set hardened/linux/amd64/no-multilib
+eend $? "Failed" || exit $?
+
+ebegin "Setting bootstrap flags"
+mkdir -p /etc/portage/package.{accept_keywords,keywords,use,env} \
+    && cat <<'EOF'>> /etc/portage/package.keywords/bootstrap
+=app-admin/salt-2015.8.8 ~amd64
+net-libs/zeromq:0/5 ~amd64
+<dev-python/pyzmq-16 ~amd64
+dev-python/cffi ~amd64
+EOF
+eend $? "Failed" || exit $?
+
+ebegin "Rebuilding openssl and openssh -bindist"
+FEATURES="-getbinpkg" ${EMERGE} --verbose=n openssl openssh
+eend $? "Failed" || exit $?
+
+ebegin "Uncommenting GENTOO_MIRRORS and other vars in make.conf"
+sed -i "s|\# sed-remove||g" /etc/portage/make.conf
+eend $? "Failed" || exit $?
+
+ebegin "Emerging git, salt qemacs nvi"
+${EMERGE} --verbose=n ">=zeromq-4.1" salt dev-vcs/git qemacs nvi
+eend $? "Failed" || exit $?
+
+ebegin "Selecting python2.7 as default python interpreter"
+eselect python set python2.7
+eend $? "Failed" || exit $?
+
+ebegin "Selecting pager"
+eselect pager set /usr/bin/less
+eend $? "Failed" || exit $?
+
+ebegin "Updating world"
+${EMERGE} -uDN @world
+eend $? "Failed" || exit $?
+
+ebegin "Cleaning deps"
+${EMERGE} --verbose=n --depclean
+eend $? "Failed" || exit $?
+
+ebegin "Removing temporary directories and logs"
+rm -rf /var/tmp/{portage,packages,distfiles} /var/log/*.log
+eend $? "Failed" || exit $?
+
+if [ ! -d /var/salt ]; then
+    ebegin "Creating /var/salt"
+    mkdir -p /var/salt
+    eend $? || exit $?
+fi

bootstrap/portage.make.conf

@@ -0,0 +1,21 @@
+CFLAGS="-march=native -O2 -pipe -mfpmath=sse"
+CXXFLAGS="${CFLAGS}"
+CHOST="x86_64-pc-linux-gnu"
+PORTDIR="/usr/portage"
+DISTDIR="/var/tmp/distfiles"
+PKGDIR="/var/tmp/packages"
+MAKEOPTS="-j2"
+FEATURES="xattr sandbox userfetch parallel-fetch parallel-install clean-logs compress-build-logs splitdebug compressdebug fail-clean unmerge-orphans getbinpkg -news nodoc noinfo noman"
+EMERGE_DEFAULT_OPTS="--quiet-build --verbose --keep-going --binpkg-changed-deps=n"
+PORT_LOGDIR="/var/log/portage"
+# sed-remove GENTOO_MIRRORS="https://gentoo.bakka.su/gentoo-distfiles"
+# sed-remove PORTAGE_BINHOST="https://gentoo.bakka.su/gentoo-packages/amd64/corei7-avx/packages"
+PORTAGE_SSH_OPTS=""
+ACCEPT_KEYWORDS="amd64"
+PYTHON_TARGETS="python2_7"
+USE_PYTHON="2.7"
+
+CPU_FLAGS_X86="mmx mmxext sse sse2 sse3 ssse3 sse4 sse4_1 sse4_2 aes avx avx2"
+USE_SALT="smp sctp xattr ssl openssl vhosts -gnutls -tcpd -doc -examples"
+USE="${USE_SALT}"
+

shared/baka-bakka

@@ -0,0 +1 @@
+Subproject commit 317886fc086385e50dbd14c36b2a42475c5b735e

shared/id_ed25519-rbkmoney-all

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACDUdGE974bgpHITGBuGGRFBADxh0JaKwWR4je7Z0fyfzQAAAJBWg49jVoOP
+YwAAAAtzc2gtZWQyNTUxOQAAACDUdGE974bgpHITGBuGGRFBADxh0JaKwWR4je7Z0fyfzQ
+AAAED56BBIUui9IjfdyNj0tGtr8W1Ie16mCYwekvZsjXbhCtR0YT3vhuCkchMYG4YZEUEA
+PGHQlorBZHiN7tnR/J/NAAAACWN5YUB0eXBlNQECAwQ=
+-----END OPENSSH PRIVATE KEY-----

shared/portage

@@ -0,0 +1 @@
+Subproject commit 35a9b56d7825743a7dc5158b1922c9bde131f2ce