From 94fceea18694b801f4aea9d0d5eeb15d46a0aa07 Mon Sep 17 00:00:00 2001 From: Alexey S Date: Thu, 28 Apr 2022 11:42:34 +0300 Subject: [PATCH] Add GitHub actions (#1) --- .editorconfig | 8 ++ .env | 8 ++ .github/codecov.yml | 11 ++ .github/workflows/build-and-push-image.yml | 54 +++++++++ .github/workflows/build-image.yml | 43 +++++++ .github/workflows/erlang-checks.yml | 39 +++++++ .gitignore | 24 ++-- .gitmodules | 4 - Dockerfile | 42 +++++++ Dockerfile.dev | 17 +++ Dockerfile.sh | 24 ---- Jenkinsfile | 22 ---- Makefile | 130 +++++++++++++-------- README.md | 42 +++++++ apps/bender/src/bender.app.src | 4 +- apps/bender/test/bender_client.erl | 2 +- apps/bender/test/bender_tests_SUITE.erl | 7 +- apps/bender/test/generator_client.erl | 2 +- apps/bender/test/generator_tests_SUITE.erl | 4 +- build_utils | 1 - docker-compose.sh => compose.yaml | 28 ++--- config/sys.config | 47 +++++--- elvis.config | 41 +++---- rebar.config | 60 ++++++---- rebar.lock | 66 ++++------- test/machinegun/config.yaml | 4 + 26 files changed, 498 insertions(+), 236 deletions(-) create mode 100644 .editorconfig create mode 100644 .env create mode 100644 .github/codecov.yml create mode 100644 .github/workflows/build-and-push-image.yml create mode 100644 .github/workflows/build-image.yml create mode 100644 .github/workflows/erlang-checks.yml delete mode 100644 .gitmodules create mode 100644 Dockerfile create mode 100644 Dockerfile.dev delete mode 100755 Dockerfile.sh delete mode 100644 Jenkinsfile delete mode 160000 build_utils rename docker-compose.sh => compose.yaml (55%) mode change 100755 => 100644 diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..5401b79 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_size = 4 +indent_style = space +trim_trailing_whitespace = true +max_line_length = 120 diff --git a/.env b/.env new file mode 100644 index 0000000..54b37a6 --- /dev/null +++ b/.env @@ -0,0 +1,8 @@ + +# NOTE +# You SHOULD specify point releases here so that build time and run time Erlang/OTPs +# are the same. See: https://github.com/erlware/relx/pull/902 +SERVICE_NAME=bender +OTP_VERSION=24.2.0 +REBAR_VERSION=3.18 +THRIFT_VERSION=0.14.2.2 diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000..ea282a0 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,11 @@ +coverage: + status: + project: + default: + # Don't allow total coverage to drop + target: auto + threshold: 0% + patch: + default: + # Force new code to be at least 80% covered + target: 80% diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml new file mode 100644 index 0000000..b704a49 --- /dev/null +++ b/.github/workflows/build-and-push-image.yml @@ -0,0 +1,54 @@ +name: Build and push Docker image +on: + push: + branches: [master] + +env: + REGISTRY: ghcr.io + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Log in to the Container registry + uses: docker/login-action@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Construct tags / labels for an image + id: meta + uses: docker/metadata-action@v3 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }} + tags: | + type=sha + + # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#setting-an-environment-variable + - name: Update environment variables + run: grep -v '^#' .env >> $GITHUB_ENV + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + OTP_VERSION=${{ env.OTP_VERSION }} + THRIFT_VERSION=${{ env.THRIFT_VERSION }} + SERVICE_NAME=${{ env.SERVICE_NAME }} diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 0000000..5e525b7 --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,43 @@ +name: Build Docker image +on: + pull_request: + branches: ["*"] + +env: + REGISTRY: ghcr.io + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Construct tags / labels for an image + id: meta + uses: docker/metadata-action@v3 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository }} + tags: | + type=sha + + # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#setting-an-environment-variable + - name: Update environment variables + run: grep -v '^#' .env >> $GITHUB_ENV + + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + + - name: Build Docker image + uses: docker/build-push-action@v2 + with: + push: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + OTP_VERSION=${{ env.OTP_VERSION }} + THRIFT_VERSION=${{ env.THRIFT_VERSION }} + SERVICE_NAME=${{ env.SERVICE_NAME }} diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml new file mode 100644 index 0000000..4bb7abf --- /dev/null +++ b/.github/workflows/erlang-checks.yml @@ -0,0 +1,39 @@ +name: Erlang CI Checks + +on: + push: + branches: + - 'master' + - 'epic/**' + pull_request: + branches: ['**'] + +jobs: + setup: + name: Load .env + runs-on: ubuntu-latest + outputs: + otp-version: ${{ steps.otp-version.outputs.version }} + rebar-version: ${{ steps.rebar-version.outputs.version }} + thrift-version: ${{ steps.thrift-version.outputs.version }} + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - run: grep -v '^#' .env >> $GITHUB_ENV + - id: otp-version + run: echo "::set-output name=version::$OTP_VERSION" + - id: rebar-version + run: echo "::set-output name=version::$REBAR_VERSION" + - id: thrift-version + run: echo "::set-output name=version::$THRIFT_VERSION" + + run: + name: Run checks + needs: setup + uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@v1.0.1 + with: + otp-version: ${{ needs.setup.outputs.otp-version }} + rebar-version: ${{ needs.setup.outputs.rebar-version }} + use-thrift: true + thrift-version: ${{ needs.setup.outputs.thrift-version }} + run-ct-with-compose: true diff --git a/.gitignore b/.gitignore index 8ae71e1..8b14b84 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,13 @@ -# general -log +# Build artifacts /_build/ -/_checkouts/ -*~ -erl_crash.dump -.tags* -*.sublime-workspace -.edts -.DS_Store -Dockerfile -docker-compose.yml -/.idea/ +*.o *.beam -rebar3.crashdump \ No newline at end of file +*.plt + +# Run artifacts +erl_crash.dump +log + +# make stuff +/.image.* +Makefile.env diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index ca5a761..0000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "build_utils"] - path = build_utils - url = git@github.com:rbkmoney/build_utils.git - branch = master diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ec0732d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +ARG OTP_VERSION + +# Build the release +FROM docker.io/library/erlang:${OTP_VERSION} AS builder +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install thrift compiler +ARG THRIFT_VERSION +ARG TARGETARCH +RUN wget -q -O- "https://github.com/valitydev/thrift/releases/download/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}-linux-${TARGETARCH}.tar.gz" \ + | tar -xvz -C /usr/local/bin/ + +# Copy sources +RUN mkdir /build +COPY . /build/ + +# Build the release +WORKDIR /build +RUN rebar3 compile && \ + rebar3 as prod release + +# Make a runner image +FROM docker.io/library/erlang:${OTP_VERSION}-slim + +ARG SERVICE_NAME + +# Set env +ENV CHARSET=UTF-8 +ENV LANG=C.UTF-8 + +# Set runtime +WORKDIR /opt/${SERVICE_NAME} + +COPY --from=builder /build/_build/prod/rel/${SERVICE_NAME} /opt/${SERVICE_NAME} + +RUN echo "#!/bin/sh" >> /entrypoint.sh && \ + echo "exec /opt/${SERVICE_NAME}/bin/${SERVICE_NAME} foreground" >> /entrypoint.sh && \ + chmod +x /entrypoint.sh +ENTRYPOINT [] +CMD ["/entrypoint.sh"] + +EXPOSE 8022 diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 0000000..e4cfa53 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,17 @@ +ARG OTP_VERSION + +FROM docker.io/library/erlang:${OTP_VERSION} +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install thrift compiler +ARG THRIFT_VERSION +ARG TARGETARCH +RUN wget -q -O- "https://github.com/valitydev/thrift/releases/download/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}-linux-${TARGETARCH}.tar.gz" \ + | tar -xvz -C /usr/local/bin/ + +# Set env +ENV CHARSET=UTF-8 +ENV LANG=C.UTF-8 + +# Set runtime +CMD ["/bin/bash"] diff --git a/Dockerfile.sh b/Dockerfile.sh deleted file mode 100755 index 02704de..0000000 --- a/Dockerfile.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -cat < -RUN mkdir -p /var/log/bender -COPY ./_build/prod/rel/bender /opt/bender -WORKDIR /opt/bender -CMD /opt/bender/bin/bender foreground -EXPOSE 8022 -LABEL com.rbkmoney.$SERVICE_NAME.parent=$BASE_IMAGE_NAME \ - com.rbkmoney.$SERVICE_NAME.parent_tag=$BASE_IMAGE_TAG \ - com.rbkmoney.$SERVICE_NAME.build_img=build \ - com.rbkmoney.$SERVICE_NAME.build_img_tag=$BUILD_IMAGE_TAG \ - com.rbkmoney.$SERVICE_NAME.commit_id=$(git rev-parse HEAD) \ - com.rbkmoney.$SERVICE_NAME.commit_number=$(git rev-list --count HEAD) \ - com.rbkmoney.$SERVICE_NAME.branch=$( \ - if [ "HEAD" != $(git rev-parse --abbrev-ref HEAD) ]; then \ - echo $(git rev-parse --abbrev-ref HEAD); \ - elif [ -n "$BRANCH_NAME" ]; then \ - echo $BRANCH_NAME; \ - else \ - echo $(git name-rev --name-only HEAD); \ - fi) -EOF diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 0241f2a..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,22 +0,0 @@ -#!groovy -// -*- mode: groovy -*- - -def finalHook = { - runStage('store CT logs') { - archive '_build/test/logs/' - } -} - -build('bender', 'docker-host', finalHook) { - checkoutRepo() - loadBuildUtils() - - def pipeErlangService - runStage('load pipeline') { - env.JENKINS_LIB = "build_utils/jenkins_lib" - env.SH_TOOLS = "build_utils/sh" - pipeErlangService = load("${env.JENKINS_LIB}/pipeErlangService.groovy") - } - - pipeErlangService.runPipe(true, true) -} diff --git a/Makefile b/Makefile index 85d47b3..8187fac 100644 --- a/Makefile +++ b/Makefile @@ -1,76 +1,108 @@ -REBAR := $(shell which rebar3 2>/dev/null || which ./rebar3) -SUBMODULES = build_utils -SUBTARGETS = $(patsubst %,%/.git,$(SUBMODULES)) +# HINT +# Use this file to override variables here. +# For example, to run with podman put `DOCKER=podman` there. +-include Makefile.env -UTILS_PATH := build_utils -TEMPLATES_PATH := . +# NOTE +# Variables specified in `.env` file are used to pick and setup specific +# component versions, both when building a development image and when running +# CI workflows on GH Actions. This ensures that tasks run with `wc-` prefix +# (like `wc-dialyze`) are reproducible between local machine and CI runners. +DOTENV := $(shell grep -v '^\#' .env) -# Name of the service -SERVICE_NAME := bender -# Service image default tag -SERVICE_IMAGE_TAG ?= $(shell git rev-parse HEAD) -# The tag for service image to be pushed with -SERVICE_IMAGE_PUSH_TAG ?= $(SERVICE_IMAGE_TAG) +# Development images +DEV_IMAGE_TAG = $(TEST_CONTAINER_NAME)-dev +DEV_IMAGE_ID = $(file < .image.dev) -# Base image for the service -BASE_IMAGE_NAME := service-erlang -BASE_IMAGE_TAG := ef20e2ec1cb1528e9214bdeb862b15478950d5cd - -# Build image tag to be used -BUILD_IMAGE_NAME := build-erlang -BUILD_IMAGE_TAG := aaa79c2d6b597f93f5f8b724eecfc31ec2e2a23b - -CALL_ANYWHERE := all submodules rebar-update compile xref lint dialyze \ - release clean distclean check format check_format - -CALL_W_CONTAINER := $(CALL_ANYWHERE) test +DOCKER ?= docker +DOCKERCOMPOSE ?= docker-compose +DOCKERCOMPOSE_W_ENV = DEV_IMAGE_TAG=$(DEV_IMAGE_TAG) $(DOCKERCOMPOSE) +REBAR ?= rebar3 +TEST_CONTAINER_NAME ?= testrunner all: compile --include $(UTILS_PATH)/make_lib/utils_container.mk --include $(UTILS_PATH)/make_lib/utils_image.mk +.PHONY: dev-image clean-dev-image wc-shell test -.PHONY: $(CALL_W_CONTAINER) +dev-image: .image.dev -# CALL_ANYWHERE -$(SUBTARGETS): %/.git: % - git submodule update --init $< - touch $@ +.image.dev: Dockerfile.dev .env + env $(DOTENV) $(DOCKERCOMPOSE_W_ENV) build $(TEST_CONTAINER_NAME) + $(DOCKER) image ls -q -f "reference=$(DEV_IMAGE_TAG)" | head -n1 > $@ -submodules: $(SUBTARGETS) +clean-dev-image: +ifneq ($(DEV_IMAGE_ID),) + $(DOCKER) image rm -f $(DEV_IMAGE_TAG) + rm .image.dev +endif -rebar-update: - $(REBAR) update +DOCKER_WC_OPTIONS := -v $(PWD):$(PWD) --workdir $(PWD) +DOCKER_WC_EXTRA_OPTIONS ?= --rm +DOCKER_RUN = $(DOCKER) run -t $(DOCKER_WC_OPTIONS) $(DOCKER_WC_EXTRA_OPTIONS) -compile: submodules rebar-update +DOCKERCOMPOSE_RUN = $(DOCKERCOMPOSE_W_ENV) run --rm $(DOCKER_WC_OPTIONS) + +# Utility tasks + +wc-shell: dev-image + $(DOCKER_RUN) --interactive --tty $(DEV_IMAGE_TAG) + +wc-%: dev-image + $(DOCKER_RUN) $(DEV_IMAGE_TAG) make $* + +# TODO docker compose down doesn't work yet +wdeps-shell: dev-image + $(DOCKERCOMPOSE_RUN) $(TEST_CONTAINER_NAME) su; \ + $(DOCKERCOMPOSE_W_ENV) down + +wdeps-%: dev-image + $(DOCKERCOMPOSE_RUN) -T $(TEST_CONTAINER_NAME) make $*; \ + res=$$?; \ + $(DOCKERCOMPOSE_W_ENV) down; \ + exit $$res + +# Rebar tasks + +rebar-shell: + $(REBAR) shell + +compile: $(REBAR) compile -xref: submodules +xref: $(REBAR) xref lint: - elvis -V rock + $(REBAR) lint -check_format: +check-format: $(REBAR) fmt -c +dialyze: + $(REBAR) as test dialyzer + +release: + $(REBAR) as prod release + +eunit: + $(REBAR) eunit --cover + +common-test: + $(REBAR) ct --cover + +cover: + $(REBAR) covertool generate + format: $(REBAR) fmt -w -dialyze: - $(REBAR) dialyzer - -check: xref lint dialyze - -release: distclean - $(REBAR) as prod release - clean: $(REBAR) clean -distclean: +distclean: clean-build-image rm -rf _build -# CALL_W_CONTAINER -test: submodules - $(REBAR) ct +test: eunit common-test + +cover-report: + $(REBAR) cover diff --git a/README.md b/README.md index c7a7ed6..4b211a4 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,44 @@ # bender Service for binding external IDs to internal IDs + +## Building + +To build the project, run the following command: + +```bash +$ make compile +``` + +## Running + +To enter the [Erlang shell][1] with the project running, run the following command: + +```bash +$ make rebar-shell +``` + +## Development environment + +### Run in a docker container + +You can run any of the tasks defined in the Makefile from inside of a docker container (defined in `Dockerfile.dev`) by prefixing the task name with `wc-`. To successfully build the dev container you need `Docker BuildKit` enabled. This can be accomplished by either installing [docker-buildx](https://docs.docker.com/buildx/working-with-buildx/) locally, or exporting the `DOCKER_BUILDKIT=1` environment variable. + +#### Example + +* This command will run the `compile` task in a docker container: +```bash +$ make wc-compile +``` + +### Run in a docker-compose environment + +Similarly, you can run any of the tasks defined in the Makefile from inside of a docker-compose environment (defined in `docker-compose.yaml`) by prefixing the task name with `wdeps-`. To successfully build the dev container you need `Docker BuildKit` enabled (see `Run in a docker container` section). It *may* also be necessary to export a `COMPOSE_DOCKER_CLI_BUILD=1` environment variable for `docker-compose` container builds to work properly. + +#### Example + +* This command will run the `test` task in a docker-compose environment: +```bash +$ make wdeps-test +``` + +[1]: http://erlang.org/doc/man/shell.html diff --git a/apps/bender/src/bender.app.src b/apps/bender/src/bender.app.src index 8be2372..7197db8 100644 --- a/apps/bender/src/bender.app.src +++ b/apps/bender/src/bender.app.src @@ -10,9 +10,9 @@ woody, scoper, % should be before any scoper event handler usage machinery, - erl_health, prometheus, - prometheus_cowboy + prometheus_cowboy, + erl_health ]}, {mod, {bender, []}}, {env, []} diff --git a/apps/bender/test/bender_client.erl b/apps/bender/test/bender_client.erl index b8b3eea..c414b93 100644 --- a/apps/bender/test/bender_client.erl +++ b/apps/bender/test/bender_client.erl @@ -10,7 +10,7 @@ -type schema() :: bender_thrift:'GenerationSchema'(). -type user_context() :: msgpack_thrift:'Value'(). --define(RETRY_STATEGY, {linear, 5, 1000}). +-define(RETRY_STATEGY, genlib_retry:linear(5, 1000)). %%% API diff --git a/apps/bender/test/bender_tests_SUITE.erl b/apps/bender/test/bender_tests_SUITE.erl index 15c21ff..98492e0 100644 --- a/apps/bender/test/bender_tests_SUITE.erl +++ b/apps/bender/test/bender_tests_SUITE.erl @@ -29,7 +29,7 @@ -define(CONFIG(Key, C), (element(2, lists:keyfind(Key, 1, C)))). --spec all() -> [atom()]. +-spec all() -> [test_case_name() | {group, group_name()}]. all() -> [ {group, main}, @@ -109,7 +109,7 @@ init_per_testcase(_Name, C) -> Client = bender_client:new(), [{client, Client} | C]. --spec end_per_testcase(atom(), config()) -> config(). +-spec end_per_testcase(atom(), config()) -> ok. end_per_testcase(_Name, _C) -> ok. @@ -238,7 +238,8 @@ generator_init(_C) -> max_connections => 10000 } }, - {ok, _Result} = woody_client:call(Call, Options). + {ok, _Result} = woody_client:call(Call, Options), + ok. -spec retrieve_unknown_id(config()) -> ok. retrieve_unknown_id(C) -> diff --git a/apps/bender/test/generator_client.erl b/apps/bender/test/generator_client.erl index a20071b..ba685bd 100644 --- a/apps/bender/test/generator_client.erl +++ b/apps/bender/test/generator_client.erl @@ -7,7 +7,7 @@ -type schema() :: bender_thrift:'GenerationSchema'(). --define(RETRY_STATEGY, {linear, 5, 1000}). +-define(RETRY_STATEGY, genlib_retry:linear(5, 1000)). %%% API diff --git a/apps/bender/test/generator_tests_SUITE.erl b/apps/bender/test/generator_tests_SUITE.erl index 88608b3..07ff64c 100644 --- a/apps/bender/test/generator_tests_SUITE.erl +++ b/apps/bender/test/generator_tests_SUITE.erl @@ -22,7 +22,7 @@ -define(CONFIG(Key, C), (element(2, lists:keyfind(Key, 1, C)))). --spec all() -> [atom()]. +-spec all() -> [test_case_name() | {group, group_name()}]. all() -> [ {group, main} @@ -76,7 +76,7 @@ init_per_testcase(_Name, C) -> Client = generator_client:new(), [{client, Client} | C]. --spec end_per_testcase(atom(), config()) -> config(). +-spec end_per_testcase(atom(), config()) -> ok. end_per_testcase(_Name, _C) -> ok. diff --git a/build_utils b/build_utils deleted file mode 160000 index a7655bc..0000000 --- a/build_utils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a7655bc60c877a65cdfe3d9b668021d970d88a76 diff --git a/docker-compose.sh b/compose.yaml old mode 100755 new mode 100644 similarity index 55% rename from docker-compose.sh rename to compose.yaml index f1a0e36..46362bc --- a/docker-compose.sh +++ b/compose.yaml @@ -1,29 +1,29 @@ -#!/bin/bash -cat < #{path => <<"/v1/bender">>}, + bender => #{path => <<"/v1/bender">>}, generator => #{path => <<"/v1/generator">>} }}, {generator, #{ - path => <<"/v1/stateproc/bender_generator">>, - schema => machinery_mg_schema_generic, - url => <<"http://machinegun:8022/v1/automaton">>, % mandatory + path => <<"/v1/stateproc/bender_generator">>, + schema => machinery_mg_schema_generic, + % mandatory + url => <<"http://machinegun:8022/v1/automaton">>, transport_opts => #{ max_connections => 1000 } }}, {sequence, #{ - path => <<"/v1/stateproc/bender_sequence">>, - schema => machinery_mg_schema_generic, - url => <<"http://machinegun:8022/v1/automaton">>, % mandatory + path => <<"/v1/stateproc/bender_sequence">>, + schema => machinery_mg_schema_generic, + % mandatory + url => <<"http://machinegun:8022/v1/automaton">>, transport_opts => #{ max_connections => 1000 } @@ -31,14 +33,19 @@ {port, 8022}, {protocol_opts, #{ - request_timeout => 5000 % time in ms with no requests before Cowboy closes the connection + % time in ms with no requests before Cowboy closes the connection + request_timeout => 5000 }}, - {shutdown_timeout, 7000}, % time in ms before woody forces connections closing + % time in ms before woody forces connections closing + {shutdown_timeout, 7000}, {transport_opts, #{ - handshake_timeout => 5000, % timeout() | infinity, default is 5000 - max_connections => 10000, % maximum number of incoming connections, default is 1024 - num_acceptors => 100 % size of acceptors pool, default is 10 + % timeout() | infinity, default is 5000 + handshake_timeout => 5000, + % maximum number of incoming connections, default is 1024 + max_connections => 10000, + % size of acceptors pool, default is 10 + num_acceptors => 100 }}, {woody_event_handlers, [ @@ -54,9 +61,9 @@ ]}, {health_check, #{ - disk => {erl_health, disk , ["/", 99]}, - memory => {erl_health, cg_memory, [99]}, - service => {erl_health, service , [<<"bender">>]} + disk => {erl_health, disk, ["/", 99]}, + memory => {erl_health, cg_memory, [99]}, + service => {erl_health, service, [<<"bender">>]} }} ]}, @@ -69,9 +76,10 @@ config => #{ type => standard_error }, - formatter => {logger_formatter, #{ - depth => 30 - }} + formatter => + {logger_formatter, #{ + depth => 30 + }} }}, {handler, console, logger_std_h, #{ config => #{ @@ -102,7 +110,8 @@ ]}, {snowflake, [ - {max_backward_clock_moving, 1000} % 1 second + % 1 second + {max_backward_clock_moving, 1000} % {machine_id, hostname_hash} % you MUST set this option in production ]}, diff --git a/elvis.config b/elvis.config index 137f627..6ee0f9f 100644 --- a/elvis.config +++ b/elvis.config @@ -1,21 +1,22 @@ [ {elvis, [ + {verbose, true}, {config, [ #{ dirs => ["apps/*/src", "apps/*/include", "apps/*/test"], filter => "*.erl", ruleset => erl_files, rules => [ - {elvis_text_style, line_length, #{limit => 120}}, - {elvis_text_style, no_trailing_whitespace, #{ignore_empty_lines => true}}, - {elvis_style, nesting_level, #{level => 3}}, - {elvis_style, function_naming_convention, #{regex => "^([a-z][a-z0-9]*_?)*$"}}, - {elvis_style, no_if_expression, disable}, - {elvis_style, dont_repeat_yourself, #{min_complexity => 15}}, - {elvis_style, no_debug_call, #{ - ignore => [bankcard_validation_invalid_carddata, bankcard_validation_valid_carddata] - }} - ] + {elvis_text_style, line_length, #{limit => 120}}, + {elvis_text_style, no_trailing_whitespace, #{ignore_empty_lines => true}}, + {elvis_style, nesting_level, #{level => 3}}, + {elvis_style, function_naming_convention, #{regex => "^([a-z][a-z0-9]*_?)*$"}}, + {elvis_style, no_if_expression, disable}, + {elvis_style, dont_repeat_yourself, #{min_complexity => 15}}, + {elvis_style, no_debug_call, #{ + ignore => [bankcard_validation_invalid_carddata, bankcard_validation_valid_carddata] + }} + ] }, #{ dirs => ["."], @@ -32,21 +33,21 @@ filter => "rebar.config", ruleset => rebar_config, rules => [ - {elvis_text_style, line_length, #{limit => 120, skip_comments => false}}, - {elvis_text_style, no_tabs}, - {elvis_text_style, no_trailing_whitespace}, - %% Temporarily disabled till regex pattern is available - {elvis_project, no_deps_master_rebar, disable} - ] + {elvis_text_style, line_length, #{limit => 120, skip_comments => false}}, + {elvis_text_style, no_tabs}, + {elvis_text_style, no_trailing_whitespace}, + %% Temporarily disabled till regex pattern is available + {elvis_project, no_deps_master_rebar, disable} + ] }, #{ dirs => ["apps/*/src"], filter => "*.app.src", rules => [ - {elvis_text_style, line_length, #{limit => 120, skip_comments => false}}, - {elvis_text_style, no_tabs}, - {elvis_text_style, no_trailing_whitespace} - ] + {elvis_text_style, line_length, #{limit => 120, skip_comments => false}}, + {elvis_text_style, no_tabs}, + {elvis_text_style, no_trailing_whitespace} + ] } ]} ]} diff --git a/rebar.config b/rebar.config index f77ccf1..973f6af 100644 --- a/rebar.config +++ b/rebar.config @@ -26,15 +26,15 @@ %% Common project dependencies. {deps, [ - {bender_proto, {git, "https://github.com/rbkmoney/bender-proto.git", {branch, "master"}}}, - {erl_health, {git, "https://github.com/rbkmoney/erlang-health.git", {branch, "master"}}}, - {genlib, {git, "https://github.com/rbkmoney/genlib.git", {branch, "master"}}}, - {machinery, {git, "https://github.com/rbkmoney/machinery.git", {branch, "master"}}}, - {scoper, {git, "https://github.com/rbkmoney/scoper.git", {branch, "master"}}}, - {snowflake, {git, "https://github.com/rbkmoney/snowflake.git", {branch, "master"}}}, - {woody, {git, "https://github.com/rbkmoney/woody_erlang.git", {branch, "master"}}}, - {prometheus, "4.6.0"}, - {prometheus_cowboy, "0.1.8"} + {prometheus, "4.8.1"}, + {prometheus_cowboy, "0.1.8"}, + {bender_proto, {git, "https://github.com/valitydev/bender-proto.git", {branch, "master"}}}, + {erl_health, {git, "https://github.com/valitydev/erlang-health.git", {branch, "master"}}}, + {genlib, {git, "https://github.com/valitydev/genlib.git", {branch, "master"}}}, + {machinery, {git, "https://github.com/valitydev/machinery.git", {branch, "master"}}}, + {scoper, {git, "https://github.com/valitydev/scoper.git", {branch, "master"}}}, + {snowflake, {git, "https://github.com/valitydev/snowflake.git", {branch, "master"}}}, + {woody, {git, "https://github.com/valitydev/woody_erlang.git", {branch, "master"}}} ]}. %% XRef checks @@ -69,15 +69,15 @@ {profiles, [ {prod, [ {deps, [ - {how_are_you, {git, "https://github.com/rbkmoney/how_are_you.git", {ref, "2fd80134"}}}, - {woody_api_hay, {git, "https://github.com/rbkmoney/woody_api_hay.git", {ref, "4c39134cd"}}}, - {logger_logstash_formatter, - {git, "https://github.com/rbkmoney/logger_logstash_formatter.git", {ref, "87e52c755"}}}, % for introspection on production - {recon, "2.5.2"} + {recon, "2.5.2"}, + {logger_logstash_formatter, + {git, "https://github.com/valitydev/logger_logstash_formatter.git", {ref, "2c7b716"}}}, + {iosetopts, {git, "https://github.com/valitydev/iosetopts.git", {ref, "edb445c"}}} ]}, {relx, [ {release, {bender, "1.0.0"}, [ + iosetopts, % tools for introspection {recon, load}, % debugger @@ -87,8 +87,6 @@ % log formatter {logger_logstash_formatter, load}, % monitoring staff - woody_api_hay, - how_are_you, sasl, bender ]}, @@ -97,16 +95,38 @@ {mode, minimal}, {extended_start_script, true} ]} + ]}, + {test, [ + {cover_enabled, true}, + {deps, []}, + {dialyzer, [ + {plt_extra_apps, [inets, eunit, common_test]} + ]} ]} ]}. -{plugins, [ +{project_plugins, [ + {rebar3_lint, "1.0.1"}, {erlfmt, "1.0.0"}, - {rebar3_lint, "0.4.0"}, - {rebar3_thrift_compiler, {git, "https://github.com/rbkmoney/rebar3_thrift_compiler.git", {tag, "0.3.1"}}} + {covertool, "2.0.4"} ]}. {erlfmt, [ {print_width, 120}, - {files, ["apps/*/{src,include,test}/*.{hrl,erl}", "rebar.config"]} + {files, ["apps/*/{src,include,test}/*.{hrl,erl}", "rebar.config", "elvis.config", "config/sys.config"]} +]}. + +{covertool, [ + {coverdata_files, [ + "eunit.coverdata", + "ct.coverdata" + ]} +]}. + +%% NOTE +%% It is needed to solve a dependency conflict with the rebar3_lint plugin +{overrides, [ + {del, accept, [{plugins, [{rebar3_archive_plugin, "0.0.2"}]}]}, + {del, prometheus_cowboy, [{plugins, [{rebar3_archive_plugin, "0.0.1"}]}]}, + {del, prometheus_httpd, [{plugins, [{rebar3_archive_plugin, "0.0.1"}]}]} ]}. diff --git a/rebar.lock b/rebar.lock index f291e8b..bcb3498 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,12 +1,11 @@ {"1.2.0", [{<<"accept">>,{pkg,<<"accept">>,<<"0.3.5">>},2}, - {<<"bear">>,{pkg,<<"bear">>,<<"0.9.0">>},2}, {<<"bender_proto">>, - {git,"https://github.com/rbkmoney/bender-proto.git", - {ref,"75be9417ba704326c5711f935f8745d504d92935"}}, + {git,"https://github.com/valitydev/bender-proto.git", + {ref,"38ce3ffde52fb2f52a8d042e67a3e2715adb7546"}}, 0}, {<<"cache">>,{pkg,<<"cache">>,<<"2.3.3">>},1}, - {<<"certifi">>,{pkg,<<"certifi">>,<<"2.6.1">>},2}, + {<<"certifi">>,{pkg,<<"certifi">>,<<"2.8.0">>},2}, {<<"cg_mon">>, {git,"https://github.com/rbkmoney/cg_mon.git", {ref,"5a87a37694e42b6592d3b4164ae54e0e87e24e18"}}, @@ -14,31 +13,19 @@ {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.9.0">>},1}, {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.11.0">>},2}, {<<"erl_health">>, - {git,"https://github.com/rbkmoney/erlang-health.git", + {git,"https://github.com/valitydev/erlang-health.git", {ref,"5958e2f35cd4d09f40685762b82b82f89b4d9333"}}, 0}, - {<<"folsom">>, - {git,"https://github.com/folsom-project/folsom.git", - {ref,"62fd0714e6f0b4e7833880afe371a9c882ea0fc2"}}, - 1}, {<<"genlib">>, - {git,"https://github.com/rbkmoney/genlib.git", - {ref,"2bbc54d4abe0f779d57c8f5911dce64d295b1cd1"}}, + {git,"https://github.com/valitydev/genlib.git", + {ref,"82c5ff3866e3019eb347c7f1d8f1f847bed28c10"}}, 0}, {<<"gproc">>,{pkg,<<"gproc">>,<<"0.9.0">>},1}, - {<<"hackney">>,{pkg,<<"hackney">>,<<"1.17.4">>},1}, - {<<"how_are_you">>, - {git,"https://github.com/rbkmoney/how_are_you.git", - {ref,"2fd8013420328464c2c84302af2781b86577b39f"}}, - 0}, + {<<"hackney">>,{pkg,<<"hackney">>,<<"1.18.0">>},1}, {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},2}, {<<"jsx">>,{pkg,<<"jsx">>,<<"3.1.0">>},1}, - {<<"logger_logstash_formatter">>, - {git,"https://github.com/rbkmoney/logger_logstash_formatter.git", - {ref,"41e8e3cc3ba6d1f53f1f0a0c9eb07c32f0868205"}}, - 0}, {<<"machinery">>, - {git,"https://github.com/rbkmoney/machinery.git", + {git,"https://github.com/valitydev/machinery.git", {ref,"db7c94b9913451e9558afa19f2fe77bf48d391da"}}, 0}, {<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},2}, @@ -48,74 +35,71 @@ 1}, {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.2.0">>},2}, {<<"msgpack_proto">>, - {git,"https://github.com/rbkmoney/msgpack-proto.git", + {git,"https://github.com/valitydev/msgpack-proto.git", {ref,"ec15d5e854ea60c58467373077d90c2faf6273d8"}}, 1}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.1">>},2}, - {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.6.0">>},0}, + {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.8.1">>},0}, {<<"prometheus_cowboy">>,{pkg,<<"prometheus_cowboy">>,<<"0.1.8">>},0}, {<<"prometheus_httpd">>,{pkg,<<"prometheus_httpd">>,<<"2.1.11">>},1}, + {<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"0.2.1">>},1}, {<<"ranch">>,{pkg,<<"ranch">>,<<"1.8.0">>},2}, {<<"scoper">>, - {git,"https://github.com/rbkmoney/scoper.git", + {git,"https://github.com/valitydev/scoper.git", {ref,"7f3183df279bc8181efe58dafd9cae164f495e6f"}}, 0}, {<<"snowflake">>, - {git,"https://github.com/rbkmoney/snowflake.git", + {git,"https://github.com/valitydev/snowflake.git", {ref,"de159486ef40cec67074afe71882bdc7f7deab72"}}, 0}, {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.6">>},2}, {<<"thrift">>, - {git,"https://github.com/rbkmoney/thrift_erlang.git", - {ref,"846a0819d9b6d09d0c31f160e33a78dbad2067b4"}}, + {git,"https://github.com/valitydev/thrift_erlang.git", + {ref,"c280ff266ae1c1906fb0dcee8320bb8d8a4a3c75"}}, 1}, {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},2}, {<<"woody">>, - {git,"https://github.com/rbkmoney/woody_erlang.git", - {ref,"330bdcf71e99c2ea7aed424cd718939cb360ec1c"}}, - 0}, - {<<"woody_api_hay">>, - {git,"https://github.com/rbkmoney/woody_api_hay.git", - {ref,"4c39134cddaa9bf6fb8db18e7030ae64f1efb3a9"}}, + {git,"https://github.com/valitydev/woody_erlang.git", + {ref,"3ddacb9296691aa8ddad05498d1fd34b078eda75"}}, 0}]}. [ {pkg_hash,[ {<<"accept">>, <<"B33B127ABCA7CC948BBE6CAA4C263369ABF1347CFA9D8E699C6D214660F10CD1">>}, - {<<"bear">>, <<"A31CCF5361791DD5E708F4789D67E2FEF496C4F05935FC59ADC11622F834D128">>}, {<<"cache">>, <<"B23A5FE7095445A88412A6E614C933377E0137B44FFED77C9B3FEF1A731A20B2">>}, - {<<"certifi">>, <<"DBAB8E5E155A0763EEA978C913CA280A6B544BFA115633FA20249C3D396D9493">>}, + {<<"certifi">>, <<"D4FB0A6BB20B7C9C3643E22507E42F356AC090A1DCEA9AB99E27E0376D695EBA">>}, {<<"cowboy">>, <<"865DD8B6607E14CF03282E10E934023A1BD8BE6F6BACF921A7E2A96D800CD452">>}, {<<"cowlib">>, <<"0B9FF9C346629256C42EBE1EEB769A83C6CB771A6EE5960BD110AB0B9B872063">>}, {<<"gproc">>, <<"853CCB7805E9ADA25D227A157BA966F7B34508F386A3E7E21992B1B484230699">>}, - {<<"hackney">>, <<"99DA4674592504D3FB0CFEF0DB84C3BA02B4508BAE2DFF8C0108BAA0D6E0977C">>}, + {<<"hackney">>, <<"C4443D960BB9FBA6D01161D01CD81173089686717D9490E5D3606644C48D121F">>}, {<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>}, {<<"jsx">>, <<"D12516BAA0BB23A59BB35DCCAF02A1BD08243FCBB9EFE24F2D9D056CCFF71268">>}, {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>}, {<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>}, {<<"parse_trans">>, <<"16328AB840CC09919BD10DAB29E431DA3AF9E9E7E7E6F0089DD5A2D2820011D8">>}, - {<<"prometheus">>, <<"20510F381DB1CCAB818B4CF2FAC5FA6AB5CC91BC364A154399901C001465F46F">>}, + {<<"prometheus">>, <<"FA76B152555273739C14B06F09F485CF6D5D301FE4E9D31B7FF803D26025D7A0">>}, {<<"prometheus_cowboy">>, <<"CFCE0BC7B668C5096639084FCD873826E6220EA714BF60A716F5BD080EF2A99C">>}, {<<"prometheus_httpd">>, <<"F616ED9B85B536B195D94104063025A91F904A4CFC20255363F49A197D96C896">>}, + {<<"quantile_estimator">>, <<"EF50A361F11B5F26B5F16D0696E46A9E4661756492C981F7B2229EF42FF1CD15">>}, {<<"ranch">>, <<"8C7A100A139FD57F17327B6413E4167AC559FBC04CA7448E9BE9057311597A1D">>}, {<<"ssl_verify_fun">>, <<"CF344F5692C82D2CD7554F5EC8FD961548D4FD09E7D22F5B62482E5AEAEBD4B0">>}, {<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]}, {pkg_hash_ext,[ {<<"accept">>, <<"11B18C220BCC2EAB63B5470C038EF10EB6783BCB1FCDB11AA4137DEFA5AC1BB8">>}, - {<<"bear">>, <<"47F71F098F2E3CD05E124A896C5EC2F155967A2B6FF6731E0D627312CCAB7E28">>}, {<<"cache">>, <<"44516CE6FA03594D3A2AF025DD3A87BFE711000EB730219E1DDEFC816E0AA2F4">>}, - {<<"certifi">>, <<"524C97B4991B3849DD5C17A631223896272C6B0AF446778BA4675A1DFF53BB7E">>}, + {<<"certifi">>, <<"6AC7EFC1C6F8600B08D625292D4BBF584E14847CE1B6B5C44D983D273E1097EA">>}, {<<"cowboy">>, <<"2C729F934B4E1AA149AFF882F57C6372C15399A20D54F65C8D67BEF583021BDE">>}, {<<"cowlib">>, <<"2B3E9DA0B21C4565751A6D4901C20D1B4CC25CBB7FD50D91D2AB6DD287BC86A9">>}, {<<"gproc">>, <<"587E8AF698CCD3504CF4BA8D90F893EDE2B0F58CABB8A916E2BF9321DE3CF10B">>}, - {<<"hackney">>, <<"DE16FF4996556C8548D512F4DBE22DD58A587BF3332E7FD362430A7EF3986B16">>}, + {<<"hackney">>, <<"9AFCDA620704D720DB8C6A3123E9848D09C87586DC1C10479C42627B905B5C5E">>}, {<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>}, {<<"jsx">>, <<"0C5CC8FDC11B53CC25CF65AC6705AD39E54ECC56D1C22E4ADB8F5A53FB9427F3">>}, {<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>}, {<<"mimerl">>, <<"F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323">>}, {<<"parse_trans">>, <<"07CD9577885F56362D414E8C4C4E6BDF10D43A8767ABB92D24CBE8B24C54888B">>}, - {<<"prometheus">>, <<"4905FD2992F8038ECCD7AA0CD22F40637ED618C0BED1F75C05AACEC15B7545DE">>}, + {<<"prometheus">>, <<"6EDFBE928D271C7F657A6F2C46258738086584BD6CAE4A000B8B9A6009BA23A5">>}, {<<"prometheus_cowboy">>, <<"BA286BECA9302618418892D37BCD5DC669A6CC001F4EB6D6AF85FF81F3F4F34C">>}, {<<"prometheus_httpd">>, <<"0BBE831452CFDF9588538EB2F570B26F30C348ADAE5E95A7D87F35A5910BCF92">>}, + {<<"quantile_estimator">>, <<"282A8A323CA2A845C9E6F787D166348F776C1D4A41EDE63046D72D422E3DA946">>}, {<<"ranch">>, <<"49FBCFD3682FAB1F5D109351B61257676DA1A2FDBE295904176D5E521A2DDFE5">>}, {<<"ssl_verify_fun">>, <<"BDB0D2471F453C88FF3908E7686F86F9BE327D065CC1EC16FA4540197EA04680">>}, {<<"unicode_util_compat">>, <<"25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521">>}]} diff --git a/test/machinegun/config.yaml b/test/machinegun/config.yaml index c431787..9f81030 100644 --- a/test/machinegun/config.yaml +++ b/test/machinegun/config.yaml @@ -8,10 +8,14 @@ namespaces: processor: url: http://bender:8022/v1/stateproc/bender_generator pool_size: 500 + worker: + message_queue_len_limit: 1000 bender_sequence: processor: url: http://bender:8022/v1/stateproc/bender_sequence pool_size: 500 + worker: + message_queue_len_limit: 1000 storage: type: memory