TD-187: Add CI/CD (#4)

* TD-187: Add CI/CD

* Fix specs

* Fix tests

* Review fixes
This commit is contained in:
ndiezel0 2022-02-21 11:56:58 +03:00 committed by GitHub
parent 8168be02fd
commit af42521899
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 384 additions and 157 deletions

7
.dockerignore Normal file
View File

@ -0,0 +1,7 @@
/_build/
/.git/
/.github/
/.vscode/
/.idea/
erl_crash.dump
rebar3.crashdump

8
.env Normal file
View File

@ -0,0 +1,8 @@
# NOTE
# You SHOULD specify point releases here so that build time and run time Erlang/OTPs
# are the same. See: https://github.com/erlware/relx/pull/902
SERVICE_NAME=bouncer
OTP_VERSION=24.2.0
REBAR_VERSION=3.18
THRIFT_VERSION=0.14.2.2
OPA_VERSION=0.37.2

View File

@ -0,0 +1,54 @@
name: Build and push Docker image
on:
push:
branches: [master]
env:
REGISTRY: ghcr.io
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Log in to the Container registry
uses: docker/login-action@v1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Construct tags / labels for an image
id: meta
uses: docker/metadata-action@v3
with:
images: |
${{ env.REGISTRY }}/${{ github.repository }}
tags: |
type=sha
# https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#setting-an-environment-variable
- name: Update environment variables
run: grep -v '^#' .env >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Setup Buildx
uses: docker/setup-buildx-action@v1
- name: Build and push Docker image
uses: docker/build-push-action@v2
with:
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
OTP_VERSION=${{ env.OTP_VERSION }}
THRIFT_VERSION=${{ env.THRIFT_VERSION }}
SERVICE_NAME=${{ env.SERVICE_NAME }}

View File

@ -1,7 +1,5 @@
name: Build Docker image
on:
push:
branches: [master]
pull_request:
branches: ["*"]
@ -15,24 +13,31 @@ jobs:
- name: Checkout code
uses: actions/checkout@v2
- name: Log in to the Container registry
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Construct tags / labels for an image
id: meta
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
uses: docker/metadata-action@v3
with:
images: |
${{ env.REGISTRY }}/${{ github.repository }}
tags: |
type=sha
- name: Build and push Docker image
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
# https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#setting-an-environment-variable
- name: Update environment variables
run: grep -v '^#' .env >> $GITHUB_ENV
- name: Setup Buildx
uses: docker/setup-buildx-action@v1
- name: Build Docker image
uses: docker/build-push-action@v2
with:
push: ${{ github.event_name == 'push' }}
push: false
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
OTP_VERSION=${{ env.OTP_VERSION }}
THRIFT_VERSION=${{ env.THRIFT_VERSION }}
SERVICE_NAME=${{ env.SERVICE_NAME }}

39
.github/workflows/erlang-checks.yaml vendored Normal file
View File

@ -0,0 +1,39 @@
name: Erlang CI Checks
on:
push:
branches:
- 'master'
- 'epic/**'
pull_request:
branches: [ '**' ]
jobs:
setup:
name: Load .env
runs-on: ubuntu-latest
outputs:
otp-version: ${{ steps.otp-version.outputs.version }}
rebar-version: ${{ steps.rebar-version.outputs.version }}
thrift-version: ${{ steps.thrift-version.outputs.version }}
steps:
- name: Checkout repository
uses: actions/checkout@v2
- run: grep -v '^#' .env >> $GITHUB_ENV
- id: otp-version
run: echo "::set-output name=version::$OTP_VERSION"
- id: rebar-version
run: echo "::set-output name=version::$REBAR_VERSION"
- id: thrift-version
run: echo "::set-output name=version::$THRIFT_VERSION"
run:
name: Run checks
needs: setup
uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@v1.0.1
with:
otp-version: ${{ needs.setup.outputs.otp-version }}
rebar-version: ${{ needs.setup.outputs.rebar-version }}
use-thrift: true
thrift-version: ${{ needs.setup.outputs.thrift-version }}
run-ct-with-compose: true

4
.gitignore vendored
View File

@ -12,3 +12,7 @@ erl_crash.dump
rebar3.crashdump
/_checkouts/
/test/policies/bundle.tar.gz
# make stuff
/.image.*
Makefile.env

View File

@ -1,17 +1,42 @@
FROM ghcr.io/rbkmoney/build-erlang:785d48cbfa7e7f355300c08ba9edc6f0e78810cb AS builder
ARG OTP_VERSION
# Build the release
FROM docker.io/library/erlang:${OTP_VERSION} AS builder
# Install thrift compiler
ARG THRIFT_VERSION
ARG TARGETARCH
RUN wget -q -O- "https://github.com/valitydev/thrift/releases/download/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}-linux-${TARGETARCH}.tar.gz" \
| tar -xvz -C /usr/local/bin/
# Copy sources
RUN mkdir /build
COPY . /build/
# Build the release
WORKDIR /build
RUN rebar3 compile
RUN rebar3 as prod release
# Keep in sync with Erlang/OTP version in build image
FROM erlang:24.1.3.0-slim
ENV SERVICE=bouncer
# Make a runner image
FROM docker.io/library/erlang:${OTP_VERSION}-slim
ARG SERVICE_NAME
# Set env
ENV CHARSET=UTF-8
ENV LANG=C.UTF-8
COPY --from=builder /build/_build/prod/rel/${SERVICE} /opt/${SERVICE}
WORKDIR /opt/${SERVICE}
# Expose SERVICE_NAME as env so CMD expands properly on start
ENV SERVICE_NAME=${SERVICE_NAME}
# Set runtime
WORKDIR /opt/${SERVICE_NAME}
COPY --from=builder /build/_build/prod/rel/${SERVICE_NAME} /opt/${SERVICE_NAME}
ENTRYPOINT []
CMD /opt/${SERVICE}/bin/${SERVICE} foreground
CMD /opt/${SERVICE_NAME}/bin/${SERVICE_NAME} foreground
EXPOSE 8022

13
Dockerfile.dev Normal file
View File

@ -0,0 +1,13 @@
ARG OTP_VERSION
FROM docker.io/library/erlang:${OTP_VERSION}
ARG THRIFT_VERSION
ARG BUILDARCH
RUN wget -q -O- "https://github.com/valitydev/thrift/releases/download/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}-linux-${BUILDARCH}.tar.gz" \
| tar -xvz -C /usr/local/bin/
ENV CHARSET=UTF-8
ENV LANG=C.UTF-8
CMD /bin/bash

126
Makefile
View File

@ -1,44 +1,71 @@
REBAR := $(shell which rebar3 2>/dev/null || which ./rebar3)
SUBMODULES = build_utils
SUBTARGETS = $(patsubst %,%/.git,$(SUBMODULES))
# HINT
# Use this file to override variables here.
# For example, to run with podman put `DOCKER=podman` there.
-include Makefile.env
UTILS_PATH := build_utils
TEMPLATES_PATH := .
# NOTE
# Variables specified in `.env` file are used to pick and setup specific
# component versions, both when building a development image and when running
# CI workflows on GH Actions. This ensures that tasks run with `wc-` prefix
# (like `wc-dialyze`) are reproducible between local machine and CI runners.
DOTENV := $(shell grep -v '^\#' .env)
# Name of the service
SERVICE_NAME := bouncer
# Service image default tag
SERVICE_IMAGE_TAG ?= $(shell git rev-parse HEAD)
# The tag for service image to be pushed with
SERVICE_IMAGE_PUSH_TAG ?= $(SERVICE_IMAGE_TAG)
# Development images
DEV_IMAGE_TAG = $(TEST_CONTAINER_NAME)-dev
DEV_IMAGE_ID = $(file < .image.dev)
# Base image for the service
BASE_IMAGE_NAME := service-erlang
BASE_IMAGE_TAG := ef20e2ec1cb1528e9214bdeb862b15478950d5cd
BUILD_IMAGE_NAME := build-erlang
BUILD_IMAGE_TAG := aaa79c2d6b597f93f5f8b724eecfc31ec2e2a23b
CALL_ANYWHERE := \
submodules \
all compile xref lint format check_format dialyze cover release clean distclean
CALL_W_CONTAINER := $(CALL_ANYWHERE) test
.PHONY: $(CALL_W_CONTAINER) all
DOCKER ?= docker
DOCKERCOMPOSE ?= docker-compose
DOCKERCOMPOSE_W_ENV = DEV_IMAGE_TAG=$(DEV_IMAGE_TAG) $(DOCKERCOMPOSE)
REBAR ?= rebar3
TEST_CONTAINER_NAME ?= testrunner
all: compile
-include $(UTILS_PATH)/make_lib/utils_container.mk
-include $(UTILS_PATH)/make_lib/utils_image.mk
.PHONY: dev-image clean-dev-image wc-shell test
$(SUBTARGETS): %/.git: %
git submodule update --init $<
touch $@
dev-image: .image.dev
submodules: $(SUBTARGETS)
.image.dev: Dockerfile.dev .env
env $(DOTENV) $(DOCKERCOMPOSE_W_ENV) build $(TEST_CONTAINER_NAME)
$(DOCKER) image ls -q -f "reference=$(DEV_IMAGE_ID)" | head -n1 > $@
compile: submodules
clean-dev-image:
ifneq ($(DEV_IMAGE_ID),)
$(DOCKER) image rm -f $(DEV_IMAGE_TAG)
rm .image.dev
endif
DOCKER_WC_OPTIONS := -v $(PWD):$(PWD) --workdir $(PWD)
DOCKER_WC_EXTRA_OPTIONS ?= --rm
DOCKER_RUN = $(DOCKER) run -t $(DOCKER_WC_OPTIONS) $(DOCKER_WC_EXTRA_OPTIONS)
DOCKERCOMPOSE_RUN = $(DOCKERCOMPOSE_W_ENV) run --rm $(DOCKER_WC_OPTIONS) $(TEST_CONTAINER_NAME)
# Utility tasks
wc-shell: dev-image
$(DOCKER_RUN) --interactive --tty $(DEV_IMAGE_TAG)
wc-%: dev-image
$(DOCKER_RUN) $(DEV_IMAGE_TAG) make $*
wdeps-shell: dev-image
$(DOCKERCOMPOSE_RUN) su; \
$(DOCKERCOMPOSE_W_ENV) down
wdeps-%: dev-image
$(DOCKERCOMPOSE_RUN) make $*; \
res=$$?; \
$(DOCKERCOMPOSE_W_ENV) down; \
exit $$res
# Rebar tasks
rebar-shell:
$(REBAR) shell
compile:
$(REBAR) compile
xref:
@ -47,29 +74,34 @@ xref:
lint:
$(REBAR) lint
check_format:
check-format:
$(REBAR) fmt -c
dialyze:
$(REBAR) as test dialyzer
release:
$(REBAR) as prod release
eunit:
$(REBAR) eunit --cover
common-test:
$(REBAR) ct --cover
cover:
$(REBAR) covertool generate
format:
$(REBAR) fmt -w
dialyze:
$(REBAR) dialyzer
release: submodules
$(REBAR) as prod release
clean:
$(REBAR) cover -r
$(REBAR) clean
distclean:
$(REBAR) clean
distclean: clean-build-image
rm -rf _build
cover:
$(REBAR) cover
test: eunit common-test
# CALL_W_CONTAINER
test: submodules
$(REBAR) do eunit, ct
cover-report:
$(REBAR) cover

41
docker-compose.yml Normal file
View File

@ -0,0 +1,41 @@
services:
testrunner:
image: $DEV_IMAGE_TAG
build:
dockerfile: Dockerfile.dev
context: .
args:
OTP_VERSION: $OTP_VERSION
THRIFT_VERSION: $THRIFT_VERSION
volumes:
- .:$PWD
hostname: $SERVICE_NAME
working_dir: $PWD
depends_on:
opa:
condition: service_healthy
ports:
- "8022"
command: /sbin/init
opa:
image: openpolicyagent/opa:${OPA_VERSION}-debug
entrypoint: ["sh", "-c"]
ports:
- "8181"
command: ["cd /var/opa/policies &&
/opa build . &&
/opa run
--server
--addr :8181
--set decision_logs.console=true
--bundle bundle.tar.gz
"]
volumes:
- ./test/policies:/var/opa/policies:rw
healthcheck:
test: ["CMD", "wget", "-q", "--tries=1", "--spider", "http://localhost:8181/"]
interval: 5s
timeout: 1s
retries: 5

67
elvis.config Normal file
View File

@ -0,0 +1,67 @@
[
{elvis, [
{verbose, true},
{config, [
#{
dirs => ["src"],
filter => "*.erl",
ruleset => erl_files,
rules => [
% Too opionated
{elvis_style, state_record_and_type, disable},
{elvis_style, invalid_dynamic_call, #{
ignore => [
% Uses thrift reflection through `struct_info/1`.
bouncer_thrift,
% Implements parts of logger duties, including message formatting.
bouncer_audit_log
]
}},
% Readable code for building bin keys
{elvis_style, no_if_expression, #{ignore => [bouncer_gunner_metrics_event_h]}},
{elvis_style, macro_names, #{ignore => [bouncer_gunner_metrics_event_h]}}
]
},
#{
dirs => ["test"],
filter => "*.erl",
ruleset => erl_files,
rules => [
% We want to use `ct:pal/2` and friends in test code.
{elvis_style, no_debug_call, disable},
% Assert macros can trigger use of ignored binding, yet we want them for better
% readability.
{elvis_style, used_ignored_variable, disable},
% Tests are usually more comprehensible when a bit more verbose.
{elvis_style, dont_repeat_yourself, #{min_complexity => 20}},
% Too opionated
{elvis_style, state_record_and_type, disable},
{elvis_style, god_modules, #{ignore => [ct_gun_event_h]}}
]
},
#{
dirs => ["."],
filter => "Makefile",
ruleset => makefiles
},
#{
dirs => ["."],
filter => "rebar.config",
rules => [
{elvis_text_style, line_length, #{limit => 100, skip_comments => false}},
{elvis_text_style, no_tabs},
{elvis_text_style, no_trailing_whitespace}
]
},
#{
dirs => ["src"],
filter => "*.app.src",
rules => [
{elvis_text_style, line_length, #{limit => 100, skip_comments => false}},
{elvis_text_style, no_tabs},
{elvis_text_style, no_trailing_whitespace}
]
}
]}
]}
].

View File

@ -52,78 +52,15 @@
]}.
%% Helpful plugins.
{plugins, [
rebar3_lint,
{erlfmt, "1.0.0"}
{project_plugins, [
{covertool, "2.0.4"},
{erlfmt, "1.0.0"},
{rebar3_lint, "1.0.1"}
]}.
{erlfmt, [
{print_width, 100},
{files, ["{src,include,test}/*.{hrl,erl}", "rebar.config"]}
]}.
%% Linter config.
{elvis, [
#{
dirs => ["src"],
filter => "*.erl",
ruleset => erl_files,
rules => [
% Too opionated
{elvis_style, state_record_and_type, disable},
{elvis_style, invalid_dynamic_call, #{
ignore => [
% Uses thrift reflection through `struct_info/1`.
bouncer_thrift,
% Implements parts of logger duties, including message formatting.
bouncer_audit_log
]
}},
% Readable code for building bin keys
{elvis_style, no_if_expression, #{ignore => [bouncer_gunner_metrics_event_h]}},
{elvis_style, macro_names, #{ignore => [bouncer_gunner_metrics_event_h]}}
]
},
#{
dirs => ["test"],
filter => "*.erl",
ruleset => erl_files,
rules => [
% We want to use `ct:pal/2` and friends in test code.
{elvis_style, no_debug_call, disable},
% Assert macros can trigger use of ignored binding, yet we want them for better
% readability.
{elvis_style, used_ignored_variable, disable},
% Tests are usually more comprehensible when a bit more verbose.
{elvis_style, dont_repeat_yourself, #{min_complexity => 20}},
% Too opionated
{elvis_style, state_record_and_type, disable},
{elvis_style, god_modules, #{ignore => [ct_gun_event_h]}}
]
},
#{
dirs => ["."],
filter => "Makefile",
ruleset => makefiles
},
#{
dirs => ["."],
filter => "rebar.config",
rules => [
{elvis_text_style, line_length, #{limit => 100, skip_comments => false}},
{elvis_text_style, no_tabs},
{elvis_text_style, no_trailing_whitespace}
]
},
#{
dirs => ["src"],
filter => "*.app.src",
rules => [
{elvis_text_style, line_length, #{limit => 100, skip_comments => false}},
{elvis_text_style, no_tabs},
{elvis_text_style, no_trailing_whitespace}
]
}
{files, ["{src,include,test}/*.{hrl,erl}", "rebar.config", "elvis.config"]}
]}.
{elvis_output_format, colors}.
@ -180,6 +117,7 @@
{test, [
{cover_enabled, true},
{deps, []}
{deps, []},
{dialyzer, [{plt_extra_apps, [eunit, common_test, gun]}]}
]}
]}.

View File

@ -62,7 +62,7 @@ end_per_suite(C) ->
init_per_testcase(Name, C) ->
[{testcase, Name} | C].
-spec end_per_testcase(testcase_name(), config()) -> config().
-spec end_per_testcase(testcase_name(), config()) -> ok.
end_per_testcase(_Name, _C) ->
ok.
@ -157,7 +157,7 @@ write_error_fails_request(C) ->
Client = mk_client(C1),
try
ok = file:delete(Filename),
ok = file:change_mode(Dirname, 8#555),
ok = file:del_dir(Dirname),
?assertError(
% NOTE
% The `_Reason` here may be either `result_unexpected` or `result_unknown`, depending

View File

@ -28,7 +28,7 @@
-define(OPA_ENDPOINT, {?OPA_HOST, 8181}).
-define(API_RULESET_ID, "service/authz/api").
-spec all() -> [atom()].
-spec all() -> [{group, group_name()}].
all() ->
[
{group, general}
@ -108,7 +108,7 @@ stop_bouncer(C) ->
init_per_testcase(Name, C) ->
[{testcase, Name} | C].
-spec end_per_testcase(atom(), config()) -> config().
-spec end_per_testcase(atom(), config()) -> ok.
end_per_testcase(_Name, _C) ->
ok.

View File

@ -48,7 +48,7 @@
-define(OPA_ENDPOINT, {?OPA_HOST, 8181}).
-define(API_RULESET_ID, "service/authz/api").
-spec all() -> [atom()].
-spec all() -> [{group, group_name()}].
all() ->
[
{group, general},
@ -160,7 +160,7 @@ stop_bouncer(C) ->
init_per_testcase(Name, C) ->
[{testcase, Name} | C].
-spec end_per_testcase(atom(), config()) -> config().
-spec end_per_testcase(atom(), config()) -> ok.
end_per_testcase(_Name, _C) ->
ok.

View File

@ -1,6 +1,6 @@
-module(ct_gun_event_h).
-behavior(gun_event).
-behaviour(gun_event).
-export([init/2]).
-export([domain_lookup_start/2]).
@ -26,7 +26,6 @@
-export([ws_send_frame_start/2]).
-export([ws_send_frame_end/2]).
-export([protocol_changed/2]).
-export([transport_changed/2]).
-export([origin_changed/2]).
-export([cancel/2]).
-export([disconnect/2]).
@ -154,11 +153,6 @@ protocol_changed(Event, State) ->
_ = ct:pal("~p [gun] protocol changed: ~p", [self(), Event]),
State.
-spec transport_changed(gun_event:transport_changed_event(), st()) -> st().
transport_changed(Event, State) ->
_ = ct:pal("~p [gun] transport changed: ~p", [self(), Event]),
State.
-spec origin_changed(gun_event:origin_changed_event(), st()) -> st().
origin_changed(Event, State) ->
_ = ct:pal("~p [gun] origin changed: ~p", [self(), Event]),

View File

@ -72,7 +72,7 @@ mode(Proxy, Scope, Mode) when is_pid(Proxy) ->
-spec stop(proxy()) -> ok.
stop(Proxy) when is_pid(Proxy) ->
proc_lib:stop(Proxy, shutdown).
proc_lib:stop(Proxy, shutdown, 1000).
%%
@ -240,10 +240,10 @@ loop_proxy_relay(St = #proxy{insock = InSock, upsock = UpSock}) ->
ok = ranch_tcp:setopts(InSock, ?PROXY_SOCKET_OPTS),
receive
{_, InSock, Data} ->
ranch_tcp:send(UpSock, Data),
ok = ranch_tcp:send(UpSock, Data),
loop_proxy_relay(St);
{_, UpSock, Data} ->
ranch_tcp:send(InSock, Data),
ok = ranch_tcp:send(InSock, Data),
loop_proxy_relay(St);
{tcp_closed, UpSock} ->
terminate(St);

View File

@ -44,11 +44,11 @@ call(Pid, Msg) ->
%%% gen_server callbacks
-spec init(term()) -> {ok, atom()}.
-spec init(term()) -> {ok, map()}.
init(_) ->
{ok, #{}}.
-spec handle_call(term(), pid(), atom()) -> {reply, atom(), atom()}.
-spec handle_call(term(), {pid(), term()}, term()) -> {reply, term(), term()}.
handle_call({append, Key, Entry}, _From, State) ->
Entries = maps:get(Key, State, []),
State1 = maps:put(Key, [Entry | Entries], State),