From 4cdec0411ff5280899405f11f6e016546c0ad2cf Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 27 May 2022 18:46:58 +0300 Subject: [PATCH] TD-292: Add GH Actions CI workflow (#1) * Switch proto upstreams to valitydev: valitydev/bender-proto@38ce3ff valitydev/binbase-proto@9db92d9 valitydev/cds-proto@ed9f907 valitydev/damsel@d384c12 valitydev/fistful-proto@c45166d valitydev/limiter-proto@8c08550 valitydev/machinegun-proto@af57ba1 valitydev/msgpack-proto@8742c7a * Rewrite Dockerfile * Include protocols as full-fledged git repos * Add GH Actions CI workflow --- .env | 4 ++ .github/workflows/build-image.yaml | 56 +++++++++++++++++++ .gitignore | 1 - .gitmodules | 35 ++++++------ Dockerfile | 54 ++++++++++++++++++ Dockerfile.sh | 41 -------------- Jenkinsfile | 42 -------------- Makefile | 29 +--------- bender-proto | 2 +- binbase-proto | 2 +- build_utils | 1 - cds-proto | 1 + cds_proto | 1 - clone-proto-modules.sh | 43 ++++++++++++++ damsel | 2 +- fistful-proto | 1 + lib/Makefile | 12 ++-- lib/observer_cli | 2 +- limiter-proto | 2 +- machinegun-proto | 1 + mgproto | 1 - msgpack-proto | 2 +- scripts/bender/generate-sequence.sh | 2 +- scripts/cds/keyring.py | 2 +- scripts/create-manual-refund.sh | 5 +- scripts/dominant/commit-base-fixture.sh | 2 +- scripts/fail-machine.sh | 4 +- scripts/get-posting-plan-batch.sh | 4 +- scripts/get-posting-plan.sh | 6 +- scripts/hellgate/get-invoice-events.sh | 7 +-- scripts/hellgate/get-invoice-state.sh | 7 +-- scripts/hellgate/get-party-events.sh | 9 ++- scripts/hellgate/get-party-state.sh | 9 ++- scripts/lib/discover | 13 ----- ...gurate_limiter.sh => configure-limiter.sh} | 2 +- ...et_limit_config.sh => get-limit-config.sh} | 2 +- scripts/machinegun/repair-machine.sh | 2 +- scripts/make-invoice-capture-success.sh | 2 +- scripts/make-invoice-paid.sh | 2 +- ...e-payment-cancelled-after-failed-cancel.sh | 2 +- ...-payment-cancelled-after-failed-capture.sh | 2 +- ...ce-payment-captured-after-failed-cancel.sh | 2 +- ...payment-processed-after-processing-fail.sh | 2 +- scripts/make-invoice-payment-processed.sh | 2 +- scripts/make-invoice-payment-refund-failed.sh | 2 +- scripts/make-invoice-payment-refunded.sh | 2 +- .../reconcile-invoice-i-dont-know-even-how.sh | 2 +- .../reconcile-invoice-refund-but-backwards.sh | 2 +- ...truct-contractors-from-legacy-contracts.py | 4 -- scripts/refund-invoice-payment.sh | 5 +- scripts/repair-invoice.sh | 6 +- scripts/rollback-posting-plan.sh | 4 +- scripts/submit-posting-plan.sh | 4 +- 53 files changed, 239 insertions(+), 217 deletions(-) create mode 100644 .env create mode 100644 .github/workflows/build-image.yaml create mode 100644 Dockerfile delete mode 100755 Dockerfile.sh delete mode 100644 Jenkinsfile delete mode 160000 build_utils create mode 160000 cds-proto delete mode 160000 cds_proto create mode 100755 clone-proto-modules.sh create mode 160000 fistful-proto create mode 160000 machinegun-proto delete mode 160000 mgproto delete mode 100755 scripts/lib/discover rename scripts/limiter/{configurate_limiter.sh => configure-limiter.sh} (96%) rename scripts/limiter/{get_limit_config.sh => get-limit-config.sh} (88%) diff --git a/.env b/.env new file mode 100644 index 0000000..48a62a4 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +OTP_VERSION=24.2.0 +STEP_VERSION=0.19.0 +THRIFT_VERSION=0.14.2.2 +WOORL_VERSION=1.6 diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml new file mode 100644 index 0000000..292a96d --- /dev/null +++ b/.github/workflows/build-image.yaml @@ -0,0 +1,56 @@ +name: Build Docker image +on: + pull_request: + branches: ["**"] + push: + branches: [master] + +env: + REGISTRY: ghcr.io + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Construct tags / labels for an image + id: meta + uses: docker/metadata-action@v3 + with: + images: ${{ env.REGISTRY }}/${{ github.repository }} + tags: type=sha + + # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#setting-an-environment-variable + - name: Update environment variables + run: grep -v '^#' .env >> $GITHUB_ENV + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: ${{ github.event_name == 'push' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64 + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + OTP_VERSION=${{ env.OTP_VERSION }} + STEP_VERSION=${{ env.STEP_VERSION }} + THRIFT_VERSION=${{ env.THRIFT_VERSION }} + WOORL_VERSION=${{ env.WOORL_VERSION }} diff --git a/.gitignore b/.gitignore index a9b4a77..ad60ddf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -Dockerfile lib/scripts woorlrc diff --git a/.gitmodules b/.gitmodules index 1dc81d6..d25e87b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,31 +1,30 @@ -[submodule "build_utils"] - path = build_utils - url = git@github.com:rbkmoney/build_utils.git - branch = master [submodule "damsel"] path = damsel - url = git@github.com:rbkmoney/damsel.git + url = https://github.com/valitydev/damsel.git branch = master [submodule "lib/observer_cli"] path = lib/observer_cli - url = https://github.com/keynslug/observer_cli + url = https://github.com/zhongwencool/observer_cli branch = master -[submodule "mgproto"] - path = mgproto - url = git@github.com:rbkmoney/machinegun_proto -[submodule "cds_proto"] - path = cds_proto - url = git@github.com:rbkmoney/cds-proto.git +[submodule "machinegun-proto"] + path = machinegun-proto + url = https://github.com/valitydev/machinegun-proto.git +[submodule "cds-proto"] + path = cds-proto + url = https://github.com/valitydev/cds-proto.git branch = master [submodule "bender-proto"] path = bender-proto - url = git@github.com:rbkmoney/bender-proto.git -[submodule "msgpack-proto"] - path = msgpack-proto - url = git@github.com:rbkmoney/msgpack-proto.git + url = https://github.com/valitydev/bender-proto.git [submodule "binbase-proto"] path = binbase-proto - url = git@github.com:rbkmoney/binbase-proto.git + url = https://github.com/valitydev/binbase-proto.git [submodule "limiter-proto"] path = limiter-proto - url = git@github.com:rbkmoney/limiter-proto.git + url = https://github.com/valitydev/limiter-proto.git +[submodule "fistful-proto"] + path = fistful-proto + url = https://github.com/valitydev/fistful-proto.git +[submodule "msgpack-proto"] + path = msgpack-proto + url = https://github.com/valitydev/msgpack-proto.git diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c491c42 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +ARG OTP_VERSION + +FROM docker.io/library/erlang:${OTP_VERSION} AS builder + +COPY . /holmes +WORKDIR /holmes +RUN make +RUN ./clone-proto-modules.sh /repos + +FROM docker.io/library/erlang:${OTP_VERSION} + +RUN apt-get --yes update \ + && apt-get --yes --no-install-recommends install \ + curl=7.74.0-1.3+deb11u1 \ + bind9-dnsutils=1:9.16.27-1~deb11u1 \ + git=1:2.30.2-1 \ + iproute2=5.10.0-4 \ + iputils-ping=3:20210202-1 \ + iputils-tracepath=3:20210202-1 \ + less=551-2 \ + nano=5.4-2 \ + netcat-openbsd=1.217-3 \ + jq=1.6-2.1 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# step-cli +ARG STEP_VERSION +ARG TARGETARCH +RUN wget -nv -O step-cli.deb "https://dl.step.sm/gh-release/cli/docs-cli-install/v${STEP_VERSION}/step-cli_${STEP_VERSION}_${TARGETARCH}.deb" \ + && dpkg -i step-cli.deb \ + && rm -vf step-cli.deb + +# thrift +ARG THRIFT_VERSION +ARG TARGETARCH +RUN wget -nv -O- "https://github.com/valitydev/thrift/releases/download/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}-linux-${TARGETARCH}.tar.gz" \ + | tar -xvz -C /usr/local/bin/ + +# woorl +ARG WOORL_VERSION +RUN wget -nv -O- "https://github.com/valitydev/woorl/releases/download/${WOORL_VERSION}/woorl-${WOORL_VERSION}.tar.gz" \ + | tar -xvz -C /usr/local/bin/ \ + && ln -sf woorl /usr/local/bin/woorl-json + +COPY ./scripts /opt/holmes/scripts +COPY --from=builder /repos /opt/holmes/ +COPY --from=builder /holmes/lib/scripts /opt/holmes/scripts +COPY woorlrc.sample /opt/holmes/ + +WORKDIR /opt/holmes +ENV CHARSET=UTF-8 +ENV LANG=C.UTF-8 +CMD ["/usr/local/bin/epmd"] diff --git a/Dockerfile.sh b/Dockerfile.sh deleted file mode 100755 index 3a1062e..0000000 --- a/Dockerfile.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -cat < -ENV THRIFT_PORT=8022 \ - CDS=cds \ - SHUMWAY=shumway \ - HELLGATE=hellgate \ - MACHINEGUN=machinegun \ - PROXY_TINKOFF=proxy-tinkoff \ - PROXY_VTB=proxy-vtb \ - PROXY_AGENT=proxy-agent \ - PROXY_MOCKETBANK=proxy-mocketbank \ - PROXY_MOCKET_INSPECTOR=proxy-inspector \ - PROXY_PIMP=pimp -COPY ./damsel/proto /opt/holmes/damsel/proto -COPY ./cds_proto/proto /opt/holmes/cds_proto/proto -COPY ./binbase-proto/proto /opt/holmes/binbase-proto/proto -COPY ./limiter-proto/proto /opt/holmes/limiter-proto/proto -COPY ./scripts /opt/holmes/scripts -COPY ./lib/scripts /opt/holmes/scripts -CMD epmd -# A bit of magic below to get a proper branch name -# even when the HEAD is detached (Hey Jenkins! -# BRANCH_NAME is available in Jenkins env). -LABEL com.rbkmoney.$SERVICE_NAME.parent=$BASE_IMAGE_NAME \ - com.rbkmoney.$SERVICE_NAME.parent_tag=$BASE_IMAGE_TAG \ - com.rbkmoney.$SERVICE_NAME.build_img=build \ - com.rbkmoney.$SERVICE_NAME.build_img_tag=$BUILD_IMAGE_TAG \ - com.rbkmoney.$SERVICE_NAME.commit_id=$(git rev-parse HEAD) \ - com.rbkmoney.$SERVICE_NAME.commit_number=$(git rev-list --count HEAD) \ - com.rbkmoney.$SERVICE_NAME.branch=$( \ - if [ "HEAD" != $(git rev-parse --abbrev-ref HEAD) ]; then \ - echo $(git rev-parse --abbrev-ref HEAD); \ - elif [ -n "$BRANCH_NAME" ]; then \ - echo $BRANCH_NAME; \ - else \ - echo $(git name-rev --name-only HEAD); \ - fi) -WORKDIR /opt/holmes -EOF diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index f5015fa..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,42 +0,0 @@ -#!groovy - -build('image-holmes', 'docker-host') { - checkoutRepo() - loadBuildUtils() - - def pipeDefault - runStage('load pipeline') { - env.JENKINS_LIB = "build_utils/jenkins_lib" - pipeDefault = load("${env.JENKINS_LIB}/pipeDefault.groovy") - } - - pipeDefault() { - runStage('fetch submodules') { - withGithubPrivkey { - sh 'make submodules' - } - } - - runStage('compile scripts') { - withGithubPrivkey { - sh 'make wc_lib' - } - } - - runStage('build image') { - sh 'make build_image' - } - - try { - if (env.BRANCH_NAME == 'master' || env.BRANCH_NAME.startsWith('epic')) { - runStage('push image') { - sh 'make push_image' - } - } - } finally { - runStage('rm local image') { - sh 'make rm_local_image' - } - } - } -} diff --git a/Makefile b/Makefile index 4e7f02c..9ecfb50 100644 --- a/Makefile +++ b/Makefile @@ -1,31 +1,6 @@ -UTILS_PATH := build_utils -TEMPLATES_PATH := . +.PHONY: lib -# Name of the service -SERVICE_NAME := holmes -# Service image default tag -SERVICE_IMAGE_TAG ?= $(shell git rev-parse HEAD) -# The tag for service image to be pushed with -SERVICE_IMAGE_PUSH_TAG ?= $(SERVICE_IMAGE_TAG) - -# Base image for the service -BASE_IMAGE_NAME := build -BASE_IMAGE_TAG := accaf81566fd3ad14bc6eadb26375ca83e76038f - -# Build image tag to be used -BUILD_IMAGE_TAG := accaf81566fd3ad14bc6eadb26375ca83e76038f - -CALL_ANYWHERE := all submodules lib - -# Hint: 'test' might be a candidate for CALL_W_CONTAINER-only target -CALL_W_CONTAINER := $(CALL_ANYWHERE) - -.PHONY: $(CALL_W_CONTAINER) - -all: submodules - --include $(UTILS_PATH)/make_lib/utils_container.mk --include $(UTILS_PATH)/make_lib/utils_image.mk +all: submodules lib submodules: @if git submodule status | egrep -q '^[-]|^[+]'; then git submodule update --init; fi diff --git a/bender-proto b/bender-proto index dfe271b..38ce3ff 160000 --- a/bender-proto +++ b/bender-proto @@ -1 +1 @@ -Subproject commit dfe271b09a2b8e457f50e4732b905a0b846bf529 +Subproject commit 38ce3ffde52fb2f52a8d042e67a3e2715adb7546 diff --git a/binbase-proto b/binbase-proto index 410b2c2..9db92d9 160000 --- a/binbase-proto +++ b/binbase-proto @@ -1 +1 @@ -Subproject commit 410b2c241d199e3cd42a9b8b553e8aa645d6ff19 +Subproject commit 9db92d90e0e28953cdb1b30c719edb529aa86579 diff --git a/build_utils b/build_utils deleted file mode 160000 index e131872..0000000 --- a/build_utils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e1318727d4d0c3e48f5122bf3197158b6695f50e diff --git a/cds-proto b/cds-proto new file mode 160000 index 0000000..ed9f907 --- /dev/null +++ b/cds-proto @@ -0,0 +1 @@ +Subproject commit ed9f9078049ebcd1439d2ada0479fc8f33ccacf1 diff --git a/cds_proto b/cds_proto deleted file mode 160000 index 07f2b0f..0000000 --- a/cds_proto +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 07f2b0f2e61d94b5fd93c40106525a9777d3398e diff --git a/clone-proto-modules.sh b/clone-proto-modules.sh new file mode 100755 index 0000000..5fa5c80 --- /dev/null +++ b/clone-proto-modules.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +PROTO_MODULE_RE='(damsel|[a-z]+-proto)' +GIT_OPTIONS='-c advice.detachedHead=false -c init.defaultBranch=master' + +TARGET_DIR="$1" +[ -z "$TARGET_DIR" ] && { + echo "usage: $0 " + exit -1 +} + +function enumerate_proto_modules { + git ls-files --error-unmatch --stage \ + | grep '^160000' \ + | awk '{ print $4 " " $2; }' \ + | grep -E "$PROTO_MODULE_RE" +} + +function emit_clone_command { + local name="$1" + local sha1="$2" + local branch="$(git config -f .gitmodules --get "submodule.${name}.branch")" + local url="$(git config -f .gitmodules --get "submodule.${name}.url")" + local target="${TARGET_DIR}/${name}" + local git="git ${GIT_OPTIONS} -C ${target}" + + mkdir -p "${target}" + $git init + $git remote add origin "${url}" + $git fetch origin "${sha1}" + if [ -n "${branch}" ]; then + $git fetch origin "${branch}" + $git checkout -B "${branch}" "${sha1}" + $git branch --set-upstream-to "origin/${branch}" + else + $git checkout "${sha1}" + fi + +} + +enumerate_proto_modules | while read name sha1; do + emit_clone_command $name $sha1 +done diff --git a/damsel b/damsel index b7f092e..d384c12 160000 --- a/damsel +++ b/damsel @@ -1 +1 @@ -Subproject commit b7f092ef41db810b79c03ad4f1f229d79091e172 +Subproject commit d384c125d16c0204e23b0d96a6ef791244a72315 diff --git a/fistful-proto b/fistful-proto new file mode 160000 index 0000000..c45166d --- /dev/null +++ b/fistful-proto @@ -0,0 +1 @@ +Subproject commit c45166d018d36a75452c3007f704e8fd3ad1056c diff --git a/lib/Makefile b/lib/Makefile index fbc999d..e99d052 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -1,11 +1,11 @@ -REBAR := $(shell which rebar3 2>/dev/null || which ./rebar3) +REBAR := rebar3 RELDIR := $(CURDIR)/scripts -.PHONY: all observer_cli +.PHONY: all -all: observer_cli +all: $(RELDIR)/observer-cli -observer_cli: - cd $@ && $(REBAR) as inet6 escriptize +$(RELDIR)/observer-cli: observer_cli + cd $< && $(REBAR) escriptize mkdir -p $(RELDIR) - cp $@/_build/inet6/bin/observer_cli $(RELDIR)/observer-cli + cp $