From 899bd71161b4d483719987014be007d7bcf7cfc3 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 20 Jun 2022 11:17:22 +0300 Subject: [PATCH] TD-312: Update to valitydev/damsel@dac2cb5 (#12) * Drop proprietary CI stuff * Sync with valitydev/erlang-templates@da1971c * Update to valitydev/dmt-core@7584133 --- .editorconfig | 8 ++ .env | 3 + .github/workflows/erlang-checks.yml | 39 +++++++++ .gitignore | 23 +++--- .gitmodules | 3 - Dockerfile.dev | 17 ++++ Jenkinsfile | 22 ----- Makefile | 119 +++++++++++++++++++--------- README.md | 44 ++++++++++ build_utils | 1 - docker-compose.sh => compose.yaml | 37 +++++---- elvis.config | 36 ++++----- rebar.config | 40 ++++++---- rebar.lock | 6 +- src/dmt_client.erl | 58 +++++++------- src/dmt_client_api.erl | 4 +- src/dmt_client_cache.erl | 59 +++++++------- test/dmt_client_cache_SUITE.erl | 57 ++++++------- test/dmt_client_fixtures.erl | 29 +++---- test/dmt_client_tests_SUITE.erl | 29 ++++--- 20 files changed, 389 insertions(+), 245 deletions(-) create mode 100644 .editorconfig create mode 100644 .env create mode 100644 .github/workflows/erlang-checks.yml delete mode 100644 .gitmodules create mode 100644 Dockerfile.dev delete mode 100644 Jenkinsfile create mode 100644 README.md delete mode 160000 build_utils rename docker-compose.sh => compose.yaml (59%) diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..5401b79 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_size = 4 +indent_style = space +trim_trailing_whitespace = true +max_line_length = 120 diff --git a/.env b/.env new file mode 100644 index 0000000..d471f87 --- /dev/null +++ b/.env @@ -0,0 +1,3 @@ +OTP_VERSION=24.3.4 +REBAR_VERSION=3.18 +THRIFT_VERSION=0.14.2.3 diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml new file mode 100644 index 0000000..79ce0b7 --- /dev/null +++ b/.github/workflows/erlang-checks.yml @@ -0,0 +1,39 @@ +name: Erlang CI Checks + +on: + push: + branches: + - 'master' + - 'epic/**' + pull_request: + branches: ['**'] + +jobs: + setup: + name: Load .env + runs-on: ubuntu-latest + outputs: + otp-version: ${{ steps.otp-version.outputs.version }} + rebar-version: ${{ steps.rebar-version.outputs.version }} + thrift-version: ${{ steps.thrift-version.outputs.version }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - run: grep -v '^#' .env >> $GITHUB_ENV + - id: otp-version + run: echo "::set-output name=version::$OTP_VERSION" + - id: rebar-version + run: echo "::set-output name=version::$REBAR_VERSION" + - id: thrift-version + run: echo "::set-output name=version::$THRIFT_VERSION" + + run: + name: Run checks + needs: setup + uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@v1.0.3 + with: + otp-version: ${{ needs.setup.outputs.otp-version }} + rebar-version: ${{ needs.setup.outputs.rebar-version }} + use-thrift: true + thrift-version: ${{ needs.setup.outputs.thrift-version }} + run-ct-with-compose: true diff --git a/.gitignore b/.gitignore index 7000a93..8b14b84 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,13 @@ -# general -log +# Build artifacts /_build/ -/.rebar3/ -/ebin/ -*~ +*.o +*.beam +*.plt + +# Run artifacts erl_crash.dump -.tags* -*.sublime-workspace -.DS_Store +log -# builtils -docker-compose.yml - -src/dmt_client_*_thrift.erl -include/dmt_client_*_thrift.hrl +# make stuff +/.image.* +Makefile.env diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 4a5266f..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "build_utils"] - path = build_utils - url = git@github.com:rbkmoney/build_utils.git diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 0000000..e4cfa53 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,17 @@ +ARG OTP_VERSION + +FROM docker.io/library/erlang:${OTP_VERSION} +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install thrift compiler +ARG THRIFT_VERSION +ARG TARGETARCH +RUN wget -q -O- "https://github.com/valitydev/thrift/releases/download/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}-linux-${TARGETARCH}.tar.gz" \ + | tar -xvz -C /usr/local/bin/ + +# Set env +ENV CHARSET=UTF-8 +ENV LANG=C.UTF-8 + +# Set runtime +CMD ["/bin/bash"] diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 64af1fb..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,22 +0,0 @@ -#!groovy -// -*- mode: groovy -*- - -def finalHook = { - runStage('store CT logs') { - archive '_build/test/logs/' - } -} - -build('dmt_client', 'docker-host', finalHook) { - checkoutRepo() - loadBuildUtils("build_utils") - - def pipeErlangLib - runStage('load pipeline') { - env.JENKINS_LIB = "build_utils/jenkins_lib" - env.SH_TOOLS = "build_utils/sh" - pipeErlangLib = load("${env.JENKINS_LIB}/pipeErlangLib.groovy") - } - - pipeErlangLib.runPipe(true, false, 'dialyze') -} diff --git a/Makefile b/Makefile index 2123ecd..623fda4 100644 --- a/Makefile +++ b/Makefile @@ -1,61 +1,108 @@ -REBAR := $(shell which rebar3 2>/dev/null || which ./rebar3) -SUBMODULES = build_utils -SUBTARGETS = $(patsubst %,%/.git,$(SUBMODULES)) +# HINT +# Use this file to override variables here. +# For example, to run with podman put `DOCKER=podman` there. +-include Makefile.env -UTILS_PATH := build_utils -TEMPLATES_PATH := . +# NOTE +# Variables specified in `.env` file are used to pick and setup specific +# component versions, both when building a development image and when running +# CI workflows on GH Actions. This ensures that tasks run with `wc-` prefix +# (like `wc-dialyze`) are reproducible between local machine and CI runners. +DOTENV := $(shell grep -v '^\#' .env) -# Name of the service -SERVICE_NAME := dmt_client +# Development images +DEV_IMAGE_TAG = dmt-client-dev +DEV_IMAGE_ID = $(file < .image.dev) -# Build image tag to be used -BUILD_IMAGE_NAME := build-erlang -BUILD_IMAGE_TAG := eb6f9920868599f7e1a8ee9aaedb1921a027f7a0 - -CALL_ANYWHERE := all submodules rebar-update compile xref lint dialyze check clean distclean check_format format -CALL_W_CONTAINER := $(CALL_ANYWHERE) test +DOCKER ?= docker +DOCKERCOMPOSE ?= docker-compose +DOCKERCOMPOSE_W_ENV = DEV_IMAGE_TAG=$(DEV_IMAGE_TAG) $(DOCKERCOMPOSE) +REBAR ?= rebar3 +TEST_CONTAINER_NAME ?= testrunner all: compile --include $(UTILS_PATH)/make_lib/utils_container.mk +.PHONY: dev-image clean-dev-image wc-shell test -.PHONY: $(CALL_W_CONTAINER) +dev-image: .image.dev -$(SUBTARGETS): %/.git: % - git submodule update --init $< - touch $@ +.image.dev: Dockerfile.dev .env + env $(DOTENV) $(DOCKERCOMPOSE_W_ENV) build $(TEST_CONTAINER_NAME) + $(DOCKER) image ls -q -f "reference=$(DEV_IMAGE_TAG)" | head -n1 > $@ -submodules: $(SUBTARGETS) +clean-dev-image: +ifneq ($(DEV_IMAGE_ID),) + $(DOCKER) image rm -f $(DEV_IMAGE_TAG) + rm .image.dev +endif -rebar-update: - $(REBAR) update +DOCKER_WC_OPTIONS := -v $(PWD):$(PWD) --workdir $(PWD) +DOCKER_WC_EXTRA_OPTIONS ?= --rm +DOCKER_RUN = $(DOCKER) run -t $(DOCKER_WC_OPTIONS) $(DOCKER_WC_EXTRA_OPTIONS) -compile: submodules rebar-update +DOCKERCOMPOSE_RUN = $(DOCKERCOMPOSE_W_ENV) run --rm $(DOCKER_WC_OPTIONS) + +# Utility tasks + +wc-shell: dev-image + $(DOCKER_RUN) --interactive --tty $(DEV_IMAGE_TAG) + +wc-%: dev-image + $(DOCKER_RUN) $(DEV_IMAGE_TAG) make $* + +# TODO docker compose down doesn't work yet +wdeps-shell: dev-image + $(DOCKERCOMPOSE_RUN) $(TEST_CONTAINER_NAME) su; \ + $(DOCKERCOMPOSE_W_ENV) down + +wdeps-%: dev-image + $(DOCKERCOMPOSE_RUN) -T $(TEST_CONTAINER_NAME) make $*; \ + res=$$?; \ + $(DOCKERCOMPOSE_W_ENV) down; \ + exit $$res + +# Rebar tasks + +rebar-shell: + $(REBAR) shell + +compile: $(REBAR) compile -xref: submodules +xref: $(REBAR) xref lint: - elvis rock + $(REBAR) lint -check_format: +check-format: $(REBAR) fmt -c +dialyze: + $(REBAR) as test dialyzer + +release: + $(REBAR) as prod release + +eunit: + $(REBAR) eunit --cover + +common-test: + $(REBAR) ct --cover + +cover: + $(REBAR) covertool generate + format: $(REBAR) fmt -w -dialyze: submodules - $(REBAR) as dialyze dialyzer - -test: submodules - $(REBAR) eunit - $(REBAR) ct - -check: lint xref dialyze - clean: $(REBAR) clean -distclean: - rm -rfv _build +distclean: clean-build-image + rm -rf _build + +test: eunit common-test + +cover-report: + $(REBAR) cover diff --git a/README.md b/README.md new file mode 100644 index 0000000..8618bbb --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# dmt-client + +The [dominant](https://github.com/valitydev/dominant) client with rich API and caching. + +## Building + +To build the project, run the following command: + +```bash +$ make compile +``` + +## Running + +To enter the [Erlang shell][1] with the project running, run the following command: + +```bash +$ make rebar-shell +``` + +## Development environment + +### Run in a docker container + +You can run any of the tasks defined in the Makefile from inside of a docker container (defined in `Dockerfile.dev`) by prefixing the task name with `wc-`. To successfully build the dev container you need `Docker BuildKit` enabled. This can be accomplished by either installing [docker-buildx](https://docs.docker.com/buildx/working-with-buildx/) locally, or exporting the `DOCKER_BUILDKIT=1` environment variable. + +#### Example + +* This command will run the `compile` task in a docker container: +```bash +$ make wc-compile +``` +#### Example + +* This command will run the `compile` task in a docker container: +```bash +$ make wc-compile +``` + +## Documentation + +@TODO Please write a couple of words about what your project does and how it does it. + +[1]: http://erlang.org/doc/man/shell.html diff --git a/build_utils b/build_utils deleted file mode 160000 index be44d69..0000000 --- a/build_utils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit be44d69fc87b22a0bb82d98d6eae7658d1647f98 diff --git a/docker-compose.sh b/compose.yaml similarity index 59% rename from docker-compose.sh rename to compose.yaml index 8ca172d..a75f97e 100755 --- a/docker-compose.sh +++ b/compose.yaml @@ -1,37 +1,42 @@ -#!/bin/bash -cat < ["src", "test"], + dirs => ["src", "include", "test"], filter => "*.erl", + ruleset => erl_files, rules => [ - {elvis_text_style, line_length, #{limit => 120, skip_comments => false}}, - {elvis_text_style, no_tabs}, - {elvis_text_style, no_trailing_whitespace}, - {elvis_style, macro_module_names}, - {elvis_style, operator_spaces, #{rules => [{right, ","}, {right, "++"}, {left, "++"}]}}, + {elvis_text_style, line_length, #{limit => 120}}, {elvis_style, nesting_level, #{level => 3}}, - {elvis_style, god_modules, #{limit => 30, ignore => [dmt_client]}}, - {elvis_style, no_if_expression}, - {elvis_style, invalid_dynamic_call, #{ignore => [elvis]}}, - {elvis_style, used_ignored_variable}, - {elvis_style, no_behavior_info}, - {elvis_style, module_naming_convention, #{regex => "^([a-z][a-z0-9]*_?)*(_SUITE)?$"}}, {elvis_style, function_naming_convention, #{regex => "^([a-z][a-z0-9]*_?)*$"}}, - {elvis_style, state_record_and_type}, - {elvis_style, no_spec_with_records}, - {elvis_style, dont_repeat_yourself, #{min_complexity => 15}}, - {elvis_style, no_debug_call, #{ignore => [elvis, elvis_utils]}} + {elvis_style, no_if_expression, disable}, + {elvis_style, god_modules, #{ + ignore => [ + % Contains ALL the API this library provides + dmt_client + ] + }} ] }, #{ @@ -37,17 +32,20 @@ #{ dirs => ["."], filter => "rebar.config", + ruleset => rebar_config, rules => [ - {elvis_text_style, line_length, #{limit => 120, skip_comments => false}}, + {elvis_text_style, line_length, #{limit => 120}}, {elvis_text_style, no_tabs}, - {elvis_text_style, no_trailing_whitespace} + {elvis_text_style, no_trailing_whitespace}, + %% Temporarily disabled till regex pattern is available + {elvis_project, no_deps_master_rebar, disable} ] }, #{ dirs => ["src"], filter => "*.app.src", rules => [ - {elvis_text_style, line_length, #{limit => 120, skip_comments => false}}, + {elvis_text_style, line_length, #{limit => 120}}, {elvis_text_style, no_tabs}, {elvis_text_style, no_trailing_whitespace} ] diff --git a/rebar.config b/rebar.config index c4e9c1f..8466d25 100644 --- a/rebar.config +++ b/rebar.config @@ -1,6 +1,5 @@ %% Common project erlang options. {erl_opts, [ - % mandatory debug_info, warnings_as_errors, @@ -27,10 +26,10 @@ %% Common project dependencies. {deps, [ - {genlib, {git, "https://github.com/valitydev/genlib.git", {branch, "master"}}}, - {woody , {git, "https://github.com/valitydev/woody_erlang.git", {branch, "master"}}}, - {dmt_core, {git, "https://github.com/valitydev/dmt_core.git", {branch, "master"}}}, - {damsel, {git, "https://github.com/valitydev/damsel.git", {branch, "master"}}} + {genlib, {git, "https://github.com/valitydev/genlib.git", {branch, "master"}}}, + {woody, {git, "https://github.com/valitydev/woody_erlang.git", {branch, "master"}}}, + {dmt_core, {git, "https://github.com/valitydev/dmt-core.git", {branch, "master"}}}, + {damsel, {git, "https://github.com/valitydev/damsel.git", {branch, "master"}}} ]}. %% XRef checks @@ -40,13 +39,9 @@ deprecated_functions_calls, deprecated_functions ]}. - % at will % {xref_warnings, true}. -%% Tests -{cover_enabled, true}. - %% Dialyzer static analyzing {dialyzer, [ {warnings, [ @@ -56,27 +51,38 @@ race_conditions, unknown ]}, - {plt_extra_apps, [erl_health]}, {plt_apps, all_deps} ]}. {profiles, [ - {dialyze, [ + {test, [ + {cover_enabled, true}, {deps, [ - {erl_health, {git, "https://github.com/valitydev/erlang-health.git", {branch, "master"}}} + {erl_health, {git, "https://github.com/valitydev/erlang-health.git", {branch, "master"}}} + ]}, + {dialyzer, [ + {plt_extra_apps, [eunit, common_test, erl_health]} ]} ]} ]}. -{plugins, [ - {erlfmt, "0.8.0"} +{project_plugins, [ + {rebar3_lint, "1.0.1"}, + {erlfmt, "1.0.0"}, + {covertool, "2.0.4"} ]}. +%% Linter config. +{elvis_output_format, colors}. + {erlfmt, [ {print_width, 120}, - {files, "{src,include,test}/*.{hrl,erl}"} + {files, ["{src,include,test}/*.{hrl,erl}", "rebar.config", "elvis.config"]} ]}. -{pre_hooks, [ - {thrift, "git submodule update --init"} +{covertool, [ + {coverdata_files, [ + "eunit.coverdata", + "ct.coverdata" + ]} ]}. diff --git a/rebar.lock b/rebar.lock index ecfac02..0cb5442 100644 --- a/rebar.lock +++ b/rebar.lock @@ -5,11 +5,11 @@ {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.11.0">>},2}, {<<"damsel">>, {git,"https://github.com/valitydev/damsel.git", - {ref,"dcd92ddba44e1d4dd9902f8c96c5524353ddd82b"}}, + {ref,"dac2cb599499cc0701e60856f4092c9ab283eedf"}}, 0}, {<<"dmt_core">>, - {git,"https://github.com/valitydev/dmt_core.git", - {ref,"910e20edbe03ae4645aa3923baea8054003753b5"}}, + {git,"https://github.com/valitydev/dmt-core.git", + {ref,"75841332fe0b40a77da0c12ea8d5dbb994da8e82"}}, 0}, {<<"genlib">>, {git,"https://github.com/valitydev/genlib.git", diff --git a/src/dmt_client.erl b/src/dmt_client.erl index 702ee61..308990c 100644 --- a/src/dmt_client.erl +++ b/src/dmt_client.erl @@ -67,33 +67,32 @@ -export_type([object_type/0]). -export_type([object_filter/0]). -export_type([object_folder/1]). --export_type([object_data/0]). -export_type([domain_object/0]). +-export_type([untagged_domain_object/0]). -export_type([domain/0]). -export_type([history/0]). -export_type([opts/0]). --include_lib("damsel/include/dmsl_domain_config_thrift.hrl"). +-include_lib("damsel/include/dmsl_domain_conf_thrift.hrl"). --type ref() :: dmsl_domain_config_thrift:'Reference'(). --type vsn() :: dmsl_domain_config_thrift:'Version'(). +-type ref() :: dmsl_domain_conf_thrift:'Reference'(). +-type vsn() :: dmsl_domain_conf_thrift:'Version'(). -type version() :: vsn() | latest. --type limit() :: dmsl_domain_config_thrift:'Limit'(). --type snapshot() :: dmsl_domain_config_thrift:'Snapshot'(). --type commit() :: dmsl_domain_config_thrift:'Commit'(). +-type limit() :: dmsl_domain_conf_thrift:'Limit'(). +-type snapshot() :: dmsl_domain_conf_thrift:'Snapshot'(). +-type commit() :: dmsl_domain_conf_thrift:'Commit'(). -type object_ref() :: dmsl_domain_thrift:'Reference'(). -type object_type() :: atom(). -type object_filter() :: fun((object_type(), domain_object()) -> boolean()). -type object_folder(Acc) :: fun((object_type(), domain_object(), Acc) -> Acc). --type object_data() :: any(). -type domain_object() :: dmsl_domain_thrift:'DomainObject'(). %% HACK: this is type required for checkout_objects_by_type: %% domain_object is any object from union, tagged with it's name %% yet there's no way to extract typespecs for untagged objects -type untagged_domain_object() :: tuple(). --type versioned_object() :: dmsl_domain_config_thrift:'VersionedObject'(). +-type versioned_object() :: dmsl_domain_conf_thrift:'VersionedObject'(). -type domain() :: dmsl_domain_thrift:'Domain'(). --type history() :: dmsl_domain_config_thrift:'History'(). +-type history() :: dmsl_domain_conf_thrift:'History'(). -type opts() :: #{ transport_opts => woody_client_thrift_http_transport:transport_options(), woody_context => woody_context:ctx() @@ -101,11 +100,13 @@ %%% API --spec try_checkout_data(object_ref()) -> {ok, object_data()} | {error, object_not_found} | no_return(). +-spec try_checkout_data(object_ref()) -> + {ok, untagged_domain_object()} | {error, object_not_found} | no_return(). try_checkout_data(ObjectRef) -> try_checkout_data(get_last_version(), ObjectRef). --spec try_checkout_data(version(), object_ref()) -> {ok, object_data()} | {error, object_not_found} | no_return(). +-spec try_checkout_data(version(), object_ref()) -> + {ok, untagged_domain_object()} | {error, object_not_found} | no_return(). try_checkout_data(Version, Ref) -> case do_checkout_object(Version, Ref, #{}) of {ok, {_Type, Object}} -> @@ -162,7 +163,10 @@ checkout_versioned_object(Reference, ObjectReference) -> -spec checkout_versioned_object(version(), object_ref(), opts()) -> versioned_object() | no_return(). checkout_versioned_object(Reference, ObjectReference, Opts) -> Version = ref_to_version(Reference), - #'VersionedObject'{version = Version, object = checkout_object(Reference, ObjectReference, Opts)}. + #domain_conf_VersionedObject{ + version = Version, + object = checkout_object(Reference, ObjectReference, Opts) + }. -spec checkout_objects_by_type(object_type()) -> [untagged_domain_object()] | no_return(). checkout_objects_by_type(ObjectType) -> @@ -262,12 +266,12 @@ insert(Reference, Objects) -> -spec insert(version(), domain_object() | [domain_object()], opts()) -> vsn() | no_return(). insert(Reference, Objects, Opts) -> - Commit = #'Commit'{ + Commit = #domain_conf_Commit{ ops = [ - {insert, #'InsertOp'{ + {insert, #domain_conf_InsertOp{ object = Object }} - || Object <- Objects + || Object <- Objects ] }, commit(Reference, Commit, Opts). @@ -285,14 +289,14 @@ update(Reference, NewObjects) -> -spec update(version(), domain_object() | [domain_object()], opts()) -> vsn() | no_return(). update(Reference, NewObjects, Opts) -> Version = updating_ref_to_version(Reference), - Commit = #'Commit'{ + Commit = #domain_conf_Commit{ ops = [ - {update, #'UpdateOp'{ + {update, #domain_conf_UpdateOp{ old_object = OldObject, new_object = NewObject }} - || NewObject = {Tag, {_ObjectName, Ref, _Data}} <- NewObjects, - OldObject <- [checkout_object(Version, {Tag, Ref}, Opts)] + || NewObject = {Tag, {_ObjectName, Ref, _Data}} <- NewObjects, + OldObject <- [checkout_object(Version, {Tag, Ref}, Opts)] ] }, %% Don't need pre-commit update: done in the beginning @@ -311,7 +315,7 @@ upsert(Reference, NewObjects) -> -spec upsert(version(), domain_object() | [domain_object()], opts()) -> vsn() | no_return(). upsert(Reference, NewObjects, Opts) -> Version = updating_ref_to_version(Reference), - Commit = #'Commit'{ + Commit = #domain_conf_Commit{ ops = lists:foldl( fun(NewObject = {Tag, {ObjectName, Ref, _Data}}, Ops) -> case unwrap_find(do_checkout_object(Version, {Tag, Ref}, Opts)) of @@ -319,7 +323,7 @@ upsert(Reference, NewObjects, Opts) -> Ops; {ok, OldObject = {Tag, {ObjectName, Ref, _OldData}}} -> [ - {update, #'UpdateOp'{ + {update, #domain_conf_UpdateOp{ old_object = OldObject, new_object = NewObject }} @@ -327,7 +331,7 @@ upsert(Reference, NewObjects, Opts) -> ]; {error, object_not_found} -> [ - {insert, #'InsertOp'{ + {insert, #domain_conf_InsertOp{ object = NewObject }} | Ops @@ -353,12 +357,12 @@ remove(Reference, Objects) -> -spec remove(version(), domain_object() | [domain_object()], opts()) -> vsn() | no_return(). remove(Reference, Objects, Opts) -> - Commit = #'Commit'{ + Commit = #domain_conf_Commit{ ops = [ - {remove, #'RemoveOp'{ + {remove, #domain_conf_RemoveOp{ object = Object }} - || Object <- Objects + || Object <- Objects ] }, commit(Reference, Commit, Opts). @@ -397,7 +401,7 @@ stop(_State) -> unwrap({ok, Acc}) -> Acc; unwrap({error, {woody_error, _} = Error}) -> erlang:error(Error); unwrap({error, version_not_found = Reason}) -> erlang:error(Reason); -unwrap({error, object_not_found}) -> erlang:throw(#'ObjectNotFound'{}). +unwrap({error, object_not_found}) -> erlang:throw(#domain_conf_ObjectNotFound{}). %% Pass object_not_found as is, raising only some of errors unwrap_find({error, {woody_error, _} = Error}) -> erlang:error(Error); diff --git a/src/dmt_client_api.erl b/src/dmt_client_api.erl index 4518d3d..8fb2232 100644 --- a/src/dmt_client_api.erl +++ b/src/dmt_client_api.erl @@ -63,9 +63,9 @@ get_service_modname(ServiceName) -> {get_service_module(ServiceName), ServiceName}. get_service_module('Repository') -> - dmsl_domain_config_thrift; + dmsl_domain_conf_thrift; get_service_module('RepositoryClient') -> - dmsl_domain_config_thrift. + dmsl_domain_conf_thrift. get_event_handlers() -> genlib_app:env(dmt_client, woody_event_handlers, []). diff --git a/src/dmt_client_cache.erl b/src/dmt_client_cache.erl index b8eff3c..110005e 100644 --- a/src/dmt_client_cache.erl +++ b/src/dmt_client_cache.erl @@ -31,10 +31,10 @@ %% 50Mb by default -define(DEFAULT_MAX_MEMORY, 52428800). --include_lib("damsel/include/dmsl_domain_config_thrift.hrl"). +-include_lib("damsel/include/dmsl_domain_conf_thrift.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). --define(meta_table_opts, [ +-define(META_TABLE_OPTS, [ named_table, ordered_set, public, @@ -43,7 +43,7 @@ {keypos, #snap.vsn} ]). --define(snapshot_table_opts, [ +-define(SNAPSHOT_TABLE_OPTS, [ ordered_set, protected, {read_concurrency, true}, @@ -117,7 +117,7 @@ get_object(Version, ObjectRef, Opts) -> end. -spec get_objects_by_type(dmt_client:vsn(), dmt_client:object_type(), dmt_client:opts()) -> - {ok, [dmt_client:domain_object()]} | {error, version_not_found | woody_error()}. + {ok, [dmt_client:untagged_domain_object()]} | {error, version_not_found | woody_error()}. get_objects_by_type(Version, ObjectType, Opts) -> case ensure_version(Version, Opts) of {ok, Version} -> do_get_objects_by_type(Version, ObjectType); @@ -206,7 +206,7 @@ code_change(_OldVsn, State, _Extra) -> -spec create_tables() -> ok. create_tables() -> - ?TABLE = ets:new(?TABLE, ?meta_table_opts), + ?TABLE = ets:new(?TABLE, ?META_TABLE_OPTS), ok. build_config() -> @@ -305,12 +305,12 @@ do_fold_objects(Version, Folder, Acc) -> end. -spec put_snapshot(dmt_client:snapshot()) -> ok. -put_snapshot(#'Snapshot'{version = Version, domain = Domain}) -> +put_snapshot(#domain_conf_Snapshot{version = Version, domain = Domain}) -> case fetch_snap(Version) of {ok, _Snap} -> ok; {error, version_not_found} -> - TID = ets:new(?MODULE, [{heir, whereis(?SERVER), ok}] ++ ?snapshot_table_opts), + TID = ets:new(?MODULE, [{heir, whereis(?SERVER), ok}] ++ ?SNAPSHOT_TABLE_OPTS), true = put_domain_to_table(TID, Domain), Snap = #snap{ vsn = Version, @@ -354,7 +354,7 @@ get_all_snaps() -> ets:tab2list(?TABLE). update(From, State) -> - restart_timer(fetch_by_reference({head, #'Head'{}}, From, #{}, State)). + restart_timer(fetch_by_reference({head, #domain_conf_Head{}}, From, #{}, State)). fetch_by_reference(Reference, From, Opts, #state{waiters = Waiters} = State) -> DispatchFun = fun dispatch_reply/2, @@ -381,7 +381,7 @@ schedule_fetch(Reference, Opts) -> case fetch(Reference, Opts) of {ok, Snapshot} -> put_snapshot(Snapshot), - {ok, Snapshot#'Snapshot'.version}; + {ok, Snapshot#domain_conf_Snapshot.version}; {error, {already_fetched, Version}} -> {ok, Version}; {error, _} = Error -> @@ -397,7 +397,7 @@ fetch(Reference, Opts) -> try do_fetch(Reference, Opts) catch - throw:#'VersionNotFound'{} -> + throw:#domain_conf_VersionNotFound{} -> {error, version_not_found}; error:{woody_error, {_Source, _Class, _Details}} = Error -> {error, Error} @@ -407,7 +407,7 @@ fetch(Reference, Opts) -> {ok, dmt_client:snapshot()} | {error, {already_fetched, dmt_client:vsn()}} | no_return(). -do_fetch({head, #'Head'{}}, Opts) -> +do_fetch({head, #domain_conf_Head{}}, Opts) -> case last_version_in_cache() of {ok, OldVersion} -> case new_commits_exist(OldVersion, Opts) of @@ -420,7 +420,7 @@ do_fetch({head, #'Head'{}}, Opts) -> {error, {already_fetched, OldVersion}} end; {error, version_not_found} -> - {ok, dmt_client_backend:checkout({head, #'Head'{}}, Opts)} + {ok, dmt_client_backend:checkout({head, #domain_conf_Head{}}, Opts)} end; do_fetch(Reference, Opts) -> {ok, dmt_client_backend:checkout(Reference, Opts)}. @@ -430,7 +430,7 @@ new_commits_exist(OldVersion, Opts) -> map_size(History) > 0. update_head(Head, PullLimit, Opts) -> - FreshHistory = dmt_client_backend:pull_range(Head#'Snapshot'.version, PullLimit, Opts), + FreshHistory = dmt_client_backend:pull_range(Head#domain_conf_Snapshot.version, PullLimit, Opts), {ok, NewHead} = dmt_history:head(FreshHistory, Head), %% Received history is smaller then PullLimit => reached the top of changes @@ -458,7 +458,7 @@ build_snapshot(#snap{vsn = Version, tid = TID}) -> dmt_domain:new(), TID ), - {ok, #'Snapshot'{version = Version, domain = Domain}} + {ok, #domain_conf_Snapshot{version = Version, domain = Domain}} catch % table was deleted due to cleanup process or crash error:badarg -> @@ -552,11 +552,14 @@ timestamp() -> -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). +-include_lib("damsel/include/dmsl_domain_thrift.hrl"). + +-type testcase() :: function() | {_Loc, function()} | [testcase()] | {setup, function(), testcase()}. % dirty hack for warn_missing_spec -spec test() -> any(). --spec all_test_() -> ok. +-spec all_test_() -> testcase(). all_test_() -> {setup, @@ -585,10 +588,10 @@ cleanup() -> -spec test_cleanup() -> _. test_cleanup() -> set_cache_limits(2), - ok = put_snapshot(#'Snapshot'{version = 4, domain = dmt_domain:new()}), - ok = put_snapshot(#'Snapshot'{version = 3, domain = dmt_domain:new()}), - ok = put_snapshot(#'Snapshot'{version = 2, domain = dmt_domain:new()}), - ok = put_snapshot(#'Snapshot'{version = 1, domain = dmt_domain:new()}), + ok = put_snapshot(#domain_conf_Snapshot{version = 4, domain = dmt_domain:new()}), + ok = put_snapshot(#domain_conf_Snapshot{version = 3, domain = dmt_domain:new()}), + ok = put_snapshot(#domain_conf_Snapshot{version = 2, domain = dmt_domain:new()}), + ok = put_snapshot(#domain_conf_Snapshot{version = 1, domain = dmt_domain:new()}), cleanup(), [ #snap{vsn = 1, _ = _}, @@ -599,12 +602,12 @@ test_cleanup() -> test_last_access() -> set_cache_limits(3), % Tables already created in test_cleanup/0 - ok = put_snapshot(#'Snapshot'{version = 4, domain = dmt_domain:new()}), - ok = put_snapshot(#'Snapshot'{version = 3, domain = dmt_domain:new()}), - ok = put_snapshot(#'Snapshot'{version = 2, domain = dmt_domain:new()}), - Ref = {category, #'domain_CategoryRef'{id = 1}}, + ok = put_snapshot(#domain_conf_Snapshot{version = 4, domain = dmt_domain:new()}), + ok = put_snapshot(#domain_conf_Snapshot{version = 3, domain = dmt_domain:new()}), + ok = put_snapshot(#domain_conf_Snapshot{version = 2, domain = dmt_domain:new()}), + Ref = {category, #domain_CategoryRef{id = 1}}, {error, object_not_found} = get_object(3, Ref, #{}), - ok = put_snapshot(#'Snapshot'{version = 1, domain = dmt_domain:new()}), + ok = put_snapshot(#domain_conf_Snapshot{version = 1, domain = dmt_domain:new()}), cleanup(), [ #snap{vsn = 1, _ = _}, @@ -619,7 +622,7 @@ test_get_object() -> Cat = {_, {_, Ref, _}} = dmt_client_fixtures:fixture(category), Domain = dmt_client_fixtures:domain_insert(Cat), - ok = put_snapshot(#'Snapshot'{version = Version, domain = Domain}), + ok = put_snapshot(#domain_conf_Snapshot{version = Version, domain = Domain}), {ok, Cat} = get_object(Version, {category, Ref}, #{}). -spec test_get_object_by_type() -> _. @@ -627,11 +630,11 @@ test_get_object_by_type() -> set_cache_limits(1), Version = 6, {_, Cat1} = dmt_client_fixtures:fixture(category), - {_, Cat2} = dmt_client_fixtures:fixture(category_2), + {_, Cat2} = dmt_client_fixtures:fixture(category2), Domain = dmt_client_fixtures:domain_with_all_fixtures(), - ok = put_snapshot(#'Snapshot'{version = Version, domain = Domain}), + ok = put_snapshot(#domain_conf_Snapshot{version = Version, domain = Domain}), {ok, Objects} = get_objects_by_type(Version, category, #{}), [Cat1, Cat2] = lists:sort(Objects). @@ -643,7 +646,7 @@ test_fold() -> Domain = dmt_client_fixtures:domain_with_all_fixtures(), - ok = put_snapshot(#'Snapshot'{version = Version, domain = Domain}), + ok = put_snapshot(#domain_conf_Snapshot{version = Version, domain = Domain}), {ok, OrdSet} = fold_objects( Version, diff --git a/test/dmt_client_cache_SUITE.erl b/test/dmt_client_cache_SUITE.erl index 1b24935..d76286c 100644 --- a/test/dmt_client_cache_SUITE.erl +++ b/test/dmt_client_cache_SUITE.erl @@ -19,16 +19,17 @@ -export([woody_error/1]). -export([object_not_found/1]). --include_lib("damsel/include/dmsl_domain_config_thrift.hrl"). +-include_lib("damsel/include/dmsl_domain_conf_thrift.hrl"). +-include_lib("damsel/include/dmsl_domain_thrift.hrl"). --define(notfound_version, 1). --define(unavailable_version, 2). --define(existing_version, 42). +-define(NOTFOUND_VERSION, 1). +-define(UNAVAILABLE_VERSION, 2). +-define(EXISTING_VERSION, 42). -type config() :: [tuple()]. -type testcase_or_group() :: atom() | {group, atom()}. --define(tests_count, 100). +-define(TESTS_COUNT, 100). %%% Common test callbacks @@ -48,15 +49,15 @@ groups() -> {group, object_not_found}, {group, mixed} ]}, - {get_snapshot_success, [parallel], lists:duplicate(?tests_count, get_snapshot_success)}, - {snapshot_not_found, [parallel], lists:duplicate(?tests_count, snapshot_not_found)}, - {woody_error, [parallel], lists:duplicate(?tests_count, woody_error)}, - {object_not_found, [parallel], lists:duplicate(?tests_count, object_not_found)}, + {get_snapshot_success, [parallel], lists:duplicate(?TESTS_COUNT, get_snapshot_success)}, + {snapshot_not_found, [parallel], lists:duplicate(?TESTS_COUNT, snapshot_not_found)}, + {woody_error, [parallel], lists:duplicate(?TESTS_COUNT, woody_error)}, + {object_not_found, [parallel], lists:duplicate(?TESTS_COUNT, object_not_found)}, {mixed, [parallel, shuffle], - lists:duplicate(?tests_count, get_snapshot_success) ++ - lists:duplicate(?tests_count, snapshot_not_found) ++ - lists:duplicate(?tests_count, woody_error) ++ - lists:duplicate(?tests_count, object_not_found)} + lists:duplicate(?TESTS_COUNT, get_snapshot_success) ++ + lists:duplicate(?TESTS_COUNT, snapshot_not_found) ++ + lists:duplicate(?TESTS_COUNT, woody_error) ++ + lists:duplicate(?TESTS_COUNT, object_not_found)} ]. -spec init_per_suite(config()) -> config(). @@ -88,21 +89,21 @@ commit(Version, _Commit, _Opts) -> Version. -spec checkout(dmt_client:ref(), dmt_client:opts()) -> dmt_client:snapshot() | no_return(). -checkout({version, ?notfound_version}, _Opts) -> - erlang:throw(#'VersionNotFound'{}); -checkout({version, ?unavailable_version}, _Opts) -> +checkout({version, ?NOTFOUND_VERSION}, _Opts) -> + erlang:throw(#domain_conf_VersionNotFound{}); +checkout({version, ?UNAVAILABLE_VERSION}, _Opts) -> woody_error:raise(system, {external, resource_unavailable, <<"test">>}); -checkout({version, ?existing_version}, _Opts) -> +checkout({version, ?EXISTING_VERSION}, _Opts) -> timer:sleep(5000), - #'Snapshot'{version = ?existing_version, domain = dmt_domain:new()}; -checkout({head, #'Head'{}}, _Opts) -> + #domain_conf_Snapshot{version = ?EXISTING_VERSION, domain = dmt_domain:new()}; +checkout({head, #domain_conf_Head{}}, _Opts) -> timer:sleep(5000), - #'Snapshot'{version = ?existing_version, domain = dmt_domain:new()}. + #domain_conf_Snapshot{version = ?EXISTING_VERSION, domain = dmt_domain:new()}. -spec checkout_object(dmt_client:ref(), dmt_client:object_ref(), dmt_client:opts()) -> - dmsl_domain_thrift:'DomainObject'() | no_return(). + no_return(). checkout_object(_Reference, _ObjectReference, _Opts) -> - erlang:throw(#'ObjectNotFound'{}). + erlang:throw(#domain_conf_ObjectNotFound{}). -spec pull_range(dmt_client:vsn(), dmt_client:limit(), dmt_client:opts()) -> dmt_client:history() | no_return(). pull_range(_Version, _Limit, _Opts) -> @@ -113,18 +114,18 @@ pull_range(_Version, _Limit, _Opts) -> -spec get_snapshot_success(config()) -> any(). get_snapshot_success(_C) -> - {ok, #'Snapshot'{}} = dmt_client_cache:get(?existing_version, #{}). + {ok, #domain_conf_Snapshot{}} = dmt_client_cache:get(?EXISTING_VERSION, #{}). -spec snapshot_not_found(config()) -> any(). snapshot_not_found(_C) -> - {error, version_not_found} = dmt_client_cache:get(?notfound_version, #{}). + {error, version_not_found} = dmt_client_cache:get(?NOTFOUND_VERSION, #{}). -spec woody_error(config()) -> any(). woody_error(_C) -> - {error, {woody_error, _}} = dmt_client_cache:get(?unavailable_version, #{}). + {error, {woody_error, _}} = dmt_client_cache:get(?UNAVAILABLE_VERSION, #{}). -spec object_not_found(config()) -> any(). object_not_found(_C) -> - Ref = {category, #'domain_CategoryRef'{id = 1}}, - {error, version_not_found} = dmt_client_cache:get_object(?notfound_version, Ref, #{}), - {error, object_not_found} = dmt_client_cache:get_object(?existing_version, Ref, #{}). + Ref = {category, #domain_CategoryRef{id = 1}}, + {error, version_not_found} = dmt_client_cache:get_object(?NOTFOUND_VERSION, Ref, #{}), + {error, object_not_found} = dmt_client_cache:get_object(?EXISTING_VERSION, Ref, #{}). diff --git a/test/dmt_client_fixtures.erl b/test/dmt_client_fixtures.erl index fe673cd..598dd4e 100644 --- a/test/dmt_client_fixtures.erl +++ b/test/dmt_client_fixtures.erl @@ -4,8 +4,6 @@ -export([domain_insert/1]). -export([domain_insert/2]). -export([fixture/1]). --export([fixture_domain_object/2]). --export([fixture_object_ref/1]). -export([fixture_category_object/2]). -export([fixture_category_ref/1]). -export([fixture_currency_object/2]). @@ -13,21 +11,24 @@ -include_lib("damsel/include/dmsl_domain_thrift.hrl"). --spec domain_with_all_fixtures() -> dmt_client:domain(). +-type domain() :: dmt_client:domain(). +-type object() :: dmt_client:domain_object(). + +-spec domain_with_all_fixtures() -> domain(). domain_with_all_fixtures() -> domain_insert( [ fixture(category), - fixture(category_2), + fixture(category2), fixture(currency) ] ). --spec domain_insert([dmt_client:domain_object()]) -> dmt_client:domain(). +-spec domain_insert(object() | [object()]) -> domain(). domain_insert(Objects) -> domain_insert(Objects, dmt_domain:new()). --spec domain_insert([dmt_client:domain_object()], dmt_client:domain()) -> dmt_client:domain(). +-spec domain_insert(object() | [object()], domain()) -> domain(). domain_insert(Object, Domain) when not is_list(Object) -> domain_insert([Object], Domain); domain_insert(Objects, Domain) -> @@ -40,13 +41,13 @@ domain_insert(Objects, Domain) -> Objects ). --spec fixture(atom()) -> dmt_client:domain_object(). +-spec fixture(atom()) -> object(). fixture(ID) -> maps:get( ID, #{ category => fixture_category_object(1, <<"cat">>), - category_2 => fixture_category_object(2, <<"dog">>), + category2 => fixture_category_object(2, <<"dog">>), currency => fixture_currency_object(<<"USD">>, #{ name => <<"dog">>, numeric_code => 840, @@ -55,22 +56,14 @@ fixture(ID) -> } ). --spec fixture_domain_object(binary(), binary()) -> dmt_client:domain_object(). -fixture_domain_object(Ref, Data) -> - fixture_category_object(Ref, Data). - --spec fixture_object_ref(binary()) -> {category, dmt_client:object_ref()}. -fixture_object_ref(Ref) -> - fixture_category_ref(Ref). - --spec fixture_category_object(binary(), binary()) -> {category, dmsl_domain_thrift:'CategoryObject'()}. +-spec fixture_category_object(integer(), binary()) -> {category, dmsl_domain_thrift:'CategoryObject'()}. fixture_category_object(Ref, Data) -> {category, #'domain_CategoryObject'{ ref = #'domain_CategoryRef'{id = Ref}, data = #'domain_Category'{name = Data, description = Data} }}. --spec fixture_category_ref(binary()) -> {category, dmsl_domain_thrift:'CategoryRef'()}. +-spec fixture_category_ref(integer()) -> {category, dmsl_domain_thrift:'CategoryRef'()}. fixture_category_ref(Ref) -> {category, #'domain_CategoryRef'{id = Ref}}. diff --git a/test/dmt_client_tests_SUITE.erl b/test/dmt_client_tests_SUITE.erl index 8e8e3ae..bc1f681 100644 --- a/test/dmt_client_tests_SUITE.erl +++ b/test/dmt_client_tests_SUITE.erl @@ -1,6 +1,5 @@ -module(dmt_client_tests_SUITE). --include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). -export([all/0]). @@ -11,7 +10,7 @@ -export([inserts_updates_upserts_and_removes/1]). -export([cached_latest/1]). --include_lib("damsel/include/dmsl_domain_config_thrift.hrl"). +-include_lib("damsel/include/dmsl_domain_conf_thrift.hrl"). %% %% tests descriptions @@ -59,15 +58,18 @@ end_per_suite(C) -> %% tests -spec insert_and_all_checkouts(term()) -> _. insert_and_all_checkouts(_C) -> - Object = dmt_client_fixtures:fixture_domain_object(1, <<"InsertFixture">>), - Ref = dmt_client_fixtures:fixture_object_ref(1), - #'ObjectNotFound'{} = (catch dmt_client:checkout_object(Ref)), - #'Snapshot'{version = Version1} = dmt_client:checkout(), + Object = dmt_client_fixtures:fixture_category_object(1, <<"InsertFixture">>), + Ref = dmt_client_fixtures:fixture_category_ref(1), + #domain_conf_ObjectNotFound{} = (catch dmt_client:checkout_object(Ref)), + #domain_conf_Snapshot{version = Version1} = dmt_client:checkout(), Version2 = dmt_client:insert(Object), true = Version1 < Version2, - #'Snapshot'{version = Version2} = dmt_client:checkout(), + #domain_conf_Snapshot{version = Version2} = dmt_client:checkout(), Object = dmt_client:checkout_object(Ref), - #'VersionedObject'{version = Version2, object = Object} = dmt_client:checkout_versioned_object(latest, Ref). + #domain_conf_VersionedObject{ + version = Version2, + object = Object + } = dmt_client:checkout_versioned_object(latest, Ref). -spec inserts_updates_upserts_and_removes(term()) -> _. inserts_updates_upserts_and_removes(_C) -> @@ -84,7 +86,10 @@ inserts_updates_upserts_and_removes(_C) -> Cat1 = dmt_client:checkout_object(Cat1Ref), Version3 = dmt_client:update(Cat1Modified), - #'VersionedObject'{version = Version3, object = Cat1Modified} = dmt_client:checkout_versioned_object(Cat1Ref), + #domain_conf_VersionedObject{ + version = Version3, + object = Cat1Modified + } = dmt_client:checkout_versioned_object(Cat1Ref), Version4 = dmt_client:upsert([Cat1ModifiedAgain, Cat2]), Cat1ModifiedAgain = dmt_client:checkout_object(Cat1Ref), @@ -92,7 +97,7 @@ inserts_updates_upserts_and_removes(_C) -> Version5 = dmt_client:remove(Cat1ModifiedAgain), ?assertThrow( - #'ObjectNotFound'{}, + #domain_conf_ObjectNotFound{}, dmt_client:checkout_object(Cat1Ref) ), Cat2 = dmt_client:checkout_object(Cat2Ref), @@ -110,8 +115,8 @@ inserts_updates_upserts_and_removes(_C) -> -spec cached_latest(term()) -> _. cached_latest(_C) -> - Object = dmt_client_fixtures:fixture_domain_object(100, <<"UpstreamLatest">>), - Commit = #'Commit'{ops = [{insert, #'InsertOp'{object = Object}}]}, + Object = dmt_client_fixtures:fixture_category_object(100, <<"UpstreamLatest">>), + Commit = #domain_conf_Commit{ops = [{insert, #domain_conf_InsertOp{object = Object}}]}, %% Get around library to prevent cache update Version = dmt_client:get_last_version(),