Read memory metrics of cgroups which current process in.

This commit is contained in:
Viacheslav V. Kovalev 2014-10-07 22:01:06 +04:00
commit 5646ed4828
16 changed files with 814 additions and 0 deletions

8
.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
ebin/
.idea/
.eunit/
*.beam
*.iml
deps/
logs/
.rebar/

28
README.md Normal file
View File

@ -0,0 +1,28 @@
cg_mon
======
Simple application to extend osmon functionality when using cgroups.
Now it only supports reading memory cgroup metrics and can't automatically detect where cgroups mounted.
Usage example.
--------------
```
1> ok = application:load(cg_mon).
ok
2> ok = application:set_env(cg_mon, update_interval, 2000). %% Update metrics each 2 seconds. 1 second by default.
ok
3> ok = application:set_env(cg_mon, cgroup_root, "/directory/where/cgroups/mounted"). %% /sys/fs/cgroup by default
ok
4> ok = application:start(cg_mon).
ok
5> cg_mem_sup:rss().
1941913600
6> cg_mem_sup:usage().
4880506880
```
To be done:
1. Add support for another cgroups such as cpuacct, blkio and so on.
2. Add event notifications (for example, about excessing some limit of memory usage).
3. Add documentation.

View File

@ -0,0 +1,3 @@
1:memory:/foo
2:cpu,cpuacct:/bar
3:cpuset:/

View File

@ -0,0 +1 @@
2

View File

@ -0,0 +1,7 @@
cache 1
rss 2
mapped_file 4
pgpgin 5
pgpgout 6
pgfault 7
pgmajfault 8

View File

@ -0,0 +1 @@
1

20
include/test_utils.hrl Normal file
View File

@ -0,0 +1,20 @@
-ifdef(TEST).
-ifndef(TEST_UTILS_HRL).
-define(TEST_UTILS_HRL, ok).
-include_lib("eunit/include/eunit.hrl").
-define(
TEST_DATA_DIR(),
filename:join( code:lib_dir(cgmon), "eunit_test_data" )
).
-define(
TEST_FILE(FileName),
filename:join(?TEST_DATA_DIR(), FileName)
).
-endif.
-endif.

95
src/cg_mem_sup.erl Normal file
View File

@ -0,0 +1,95 @@
-module(cg_mem_sup).
-author("Viacheslav V. Kovalev").
%% API
-export([
provided_resource/0,
key_value_sources/0,
single_value_sources/0
]).
-export([
limit/0,
swlimit/0,
usage/0,
swusage/0,
cache/0,
rss/0,
rss_huge/0,
mapped_file/0,
pgpgin/0,
pgpgout/0,
swap/0,
writeback/0,
inactive_anon/0,
active_anon/0,
inactive_file/0,
active_file/0
]).
-define(PROVIDED_RESOURCE, memory).
-define(KEY_VALUE_SOURCES, ["memory.stat"]).
-define(SINGLE_VALUE_SOURCES, [
{limit, "memory.limit_in_bytes"},
{swlimit, "memory.memsw.limit_in_bytes"},
{usage, "memory.usage_in_bytes"},
{swusage, "memory.memsw.usage_in_bytes"}
]).
provided_resource() ->
?PROVIDED_RESOURCE.
key_value_sources() ->
?KEY_VALUE_SOURCES.
single_value_sources() ->
?SINGLE_VALUE_SOURCES.
usage() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, usage).
swusage() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, swusage).
limit() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, limit).
swlimit() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, swlimit).
cache() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, cache).
rss() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, rss).
rss_huge() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, rss_huge).
mapped_file() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, mapped_file).
pgpgin() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, pgpgin).
pgpgout() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, pgpgout).
swap() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, swap).
writeback() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, writeback).
inactive_anon() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, inactive_anon).
active_anon() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, active_anon).
inactive_file() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, inactive_file).
active_file() ->
cg_mon_reader:read_meter(?PROVIDED_RESOURCE, active_file).

14
src/cg_mon.app.src Normal file
View File

@ -0,0 +1,14 @@
{application, cg_mon,
[
{description, "Reads cgroup's metrics"},
{vsn, "0.0.1"},
{registered, []},
{applications, [
kernel,
stdlib
]},
{mod, { cg_mon_app, []}},
{env, [
{update_interval, 1000}
]}
]}.

170
src/cg_mon_app.erl Normal file
View File

@ -0,0 +1,170 @@
-module(cg_mon_app).
-author("Viacheslav V. Kovalev").
-behaviour(application).
-behaviour(supervisor).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-endif.
%% Application callbacks
-export([start/2,
stop/1]).
%% Supervisor callbacks
-export([init/1]).
%% Api functions
-export([
update_interval/0,
update_interval/1,
metrics_table/0
]).
%%%===================================================================
%%% Api functions
%%%===================================================================
update_interval() ->
{ok, Value} = application:get_env(cg_mon, update_interval),
Value.
update_interval(Interval) when is_integer(Interval), Interval > 0 ->
application:set_env(cg_mon, update_interval, Interval).
metrics_table() ->
cg_mon_metrics.
%%%===================================================================
%%% Application callbacks
%%%===================================================================
start(_StartType, _StartArgs) ->
SupervisorArgs = [
cgroup_root(), cgroup_discovery_file(), handler_modules()
],
ets:new( metrics_table(), [public, named_table] ),
SupervisorStartRes = supervisor:start_link({local, cg_mon_sup}, cg_mon_app, SupervisorArgs),
case SupervisorStartRes of
{ok, Pid} ->
{ok, Pid};
_ ->
{error, cgroups_error}
end.
stop(_State) ->
ok.
init([CgroupRoot, CgroupDiscovery, AvailableHandlers]) ->
SupFlags = {one_for_one, 5, 3600},
case cg_mon_lib:discover_cgroups( CgroupRoot, CgroupDiscovery ) of
{ok, CgroupsMap} ->
HandlersMap = orddict:from_list([
{HandlerModule:provided_resource(), HandlerModule}
|| HandlerModule <- AvailableHandlers
]),
Children =
lists:foldl(
fun({Resource, Path}, Acc) ->
case orddict:find(Resource, HandlersMap) of
{ok, HandlerModule} ->
[ child_spec(HandlerModule, Path) | Acc ];
_ ->
Acc
end
end, [], CgroupsMap
),
{ok, {SupFlags, Children}};
{error, _Reason} ->
ignore
end.
%%%===================================================================
%%% Internal functions
%%%===================================================================
child_spec(Module, FullCgroupPath) ->
Id = {Module, reader},
Args = [
Module:provided_resource(),
Module:key_value_sources(),
Module:single_value_sources(),
FullCgroupPath
],
{Id,
{cg_mon_reader, start_link, Args},
permanent, 2000, worker, [cg_mon_reader]
}.
cgroup_root() ->
case application:get_env(cg_mon, cgroup_root) of
{ok, Value} when is_list(Value) ->
Value;
_ ->
cg_mon_lib:cgroup_root()
end.
cgroup_discovery_file() ->
case application:get_env(cg_mon, cgroup_discovery_file) of
{ok, Value} when is_list(Value) ->
Value;
_ ->
cg_mon_lib:discovery_file()
end.
handler_modules() ->
[cg_mem_sup].
%%%===================================================================
%%% Unit tests
%%%===================================================================
-ifdef(TEST).
-include_lib("cgmon/include/test_utils.hrl").
reader_spec_test_() ->
Res = init([
?TEST_DATA_DIR(), ?TEST_FILE("cgroup.discovery"),
[cg_mem_sup]
]),
ExpectedArgs = [
cg_mem_sup:provided_resource(),
cg_mem_sup:key_value_sources(),
cg_mem_sup:single_value_sources(),
?TEST_FILE("memory/foo") ++ "/"
],
ExpectedChildren = [
{{cg_mem_sup, reader}, {cg_mon_reader, start_link, ExpectedArgs},
permanent, 2000, worker, [cg_mon_reader]}
],
?_assertMatch(
{ok, {_, ExpectedChildren}},
Res
).
no_discovery_test_() ->
Res = init([?TEST_DATA_DIR(), ?TEST_FILE("nonexistent.file"), [cg_mem_sup]]),
[
?_assertEqual(ignore, Res)
].
-endif.

282
src/cg_mon_lib.erl Normal file
View File

@ -0,0 +1,282 @@
-module(cg_mon_lib).
-author("Viacheslav V. Kovalev").
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-endif.
%% API
-export([
discovery_file/0,
cgroup_root/0,
discover_cgroups/0,
discover_cgroups/1,
discover_cgroups/2,
read_cgroups_metrics/5
]).
-export_type([
cgroup_map/0,
metric_source/0
]).
-type cgroup_map() :: [{Resource :: atom(), FullPath :: string()}].
-type metric_source() :: StatFile :: string() | { Key :: atom(), ValueFile :: string()}.
-spec discovery_file() -> string().
discovery_file() ->
"/proc/self/cgroup".
-spec cgroup_root() -> string().
cgroup_root() ->
"/sys/fs/cgroup".
-spec discover_cgroups() ->
{ok, cgroup_map()}
| {error, Reason :: atom()}.
discover_cgroups() ->
discover_cgroups(cgroup_root(), discovery_file()).
-spec discover_cgroups(Root :: string()) ->
{ok, cgroup_map()}
| {error, Reason :: atom()}.
discover_cgroups(Root) ->
discover_cgroups(Root, discovery_file()).
-spec discover_cgroups(Root :: string(), FileName :: string()) ->
{ok, cgroup_map()}
| {error, Reason :: atom()}.
discover_cgroups(Root, FileName) ->
with_file(
fun(DiscoveryFile) ->
parse_discovery_file(DiscoveryFile, Root)
end,
FileName
).
-spec parse_discovery_file(File :: file:io_device(), CgroupRoot :: string()) ->
{ok, cgroup_map()}
| {error, Reason :: atom()}.
parse_discovery_file(File, CgroupRoot) ->
foldl_lines(
fun(Line, Acc) ->
[_, Resources, GroupName] = string:tokens(Line, ":"),
ResourcesList = string:tokens(Resources, ","),
RelativeName = fixup_group_name(GroupName),
FullName = filename:join([CgroupRoot, Resources]) ++ RelativeName,
lists:foldl(
fun(ResourceName, TmpAcc) ->
[ {list_to_atom(ResourceName), FullName} | TmpAcc]
end, Acc, ResourcesList
)
end, [], File
).
-spec with_file(fun( (File :: file:io_device()) -> any() ), Filename :: string()) ->
any().
with_file(Fun, FileName) ->
case file:open(FileName, [read]) of
{ok, File} ->
Result = Fun(File),
file:close(File),
Result;
Error ->
Error
end.
-spec foldl_lines(fun((Line :: string(), Acc :: any()) -> any()), InitAcc :: any(), File :: file:io_device()) ->
any().
foldl_lines(Fun, Acc, File) ->
case file:read_line(File) of
{ok, Line} ->
NewAcc = Fun( string:strip(Line, right, 10), Acc ),
foldl_lines(Fun, NewAcc, File);
eof ->
{ok, Acc};
Error ->
Error
end.
-spec read_cgroups_metrics(
OutputTable :: ets:tid(),
Resource :: atom(),
FullCgroupPath :: string(),
KeyValueStore :: [ ],
SingleValueStore :: [ ]
) ->
[{ FailedSource :: metric_source(), ErrorReason :: atom() }].
read_cgroups_metrics(OutputTable, Resource, FullCgroupPath, KeyValueStore, SingleValueStore) ->
KeyValueErrors =
lists:foldl(
fun(KeyValueFile, Acc) ->
case read_stat_file(FullCgroupPath, KeyValueFile) of
{ok, KeyValueData} ->
[
ets:insert(OutputTable, {{Resource, list_to_atom(Key)}, Value})
|| {Key, Value} <- KeyValueData
],
Acc;
{error, Reason} ->
[{KeyValueFile, Reason} | Acc]
end
end, [], KeyValueStore
),
lists:foldl(
fun(Entry = {Key, SingleValueFile}, Acc) ->
FullFileName = filename:join(FullCgroupPath, SingleValueFile),
ReadResult =
with_file(
fun(File) ->
{ok, Line} = file:read_line(File),
{ok, list_to_integer( strip_line_ending(Line) )}
end,
FullFileName
),
case ReadResult of
{ok, Value} ->
ets:insert(OutputTable, {{Resource, Key}, Value}),
Acc;
{error, Reason} ->
[{Entry, Reason} | Acc]
end
end, KeyValueErrors, SingleValueStore
).
-spec read_stat_file(FullCgroupPath :: string(), StatFile :: string()) ->
[{Key :: string(), Value :: integer()}].
read_stat_file(FullCgroupPath, StatFile) ->
with_file(
fun(File) ->
foldl_lines(
fun(Line, Acc) ->
[Key, Value] = string:tokens(Line, " "),
[{Key, list_to_integer(Value)} | Acc]
end, [], File
)
end,
filename:join(FullCgroupPath, StatFile)
).
-spec fixup_group_name(GroupName :: string()) ->
string().
fixup_group_name(GroupName) ->
case strip_line_ending(GroupName) of
Value = "/" -> Value;
Value -> Value ++ "/"
end.
-spec strip_line_ending(Line :: string()) ->
string().
strip_line_ending(Line) ->
string:strip(Line, right, 10).
%% ----------------------------------------------------------------------------
%% EUnit test cases
%% ----------------------------------------------------------------------------
-ifdef(TEST).
-include_lib("cgmon/include/test_utils.hrl").
discovery_error_test() ->
Res = discover_cgroups(cgroup_root(), ?TEST_FILE("non-existent-file")),
?assertEqual(Res, {error, enoent}).
discovery_denied_test() ->
Res = discover_cgroups(cgroup_root(), "/root/some-not-permitted-file"),
?assertEqual(Res, {error, eacces}).
discovery_test_() ->
{ok, Res} = discover_cgroups(cgroup_root(), ?TEST_FILE("cgroup.discovery")),
[
?_assertEqual(4, length(Res)),
?_assertEqual("/sys/fs/cgroup/memory/foo/", proplists:get_value(memory, Res)),
?_assertEqual("/sys/fs/cgroup/cpu,cpuacct/bar/", proplists:get_value(cpu, Res)),
?_assertEqual("/sys/fs/cgroup/cpu,cpuacct/bar/", proplists:get_value(cpuacct, Res)),
?_assertEqual("/sys/fs/cgroup/cpuset/", proplists:get_value(cpuset, Res))
].
read_stat_file_error_test_() ->
[
?_assertEqual(
{error, enoent},
read_stat_file(?TEST_FILE("some/not/existed/"), "file.stat")
),
?_assertEqual(
{error, eacces},
read_stat_file("/root/some/not/permitted", "file.stat")
)
].
read_stat_file_test_() ->
{ok, Result} = read_stat_file(?TEST_FILE("memory/foo/"), "memory.stat"),
[
?_assertEqual(7, length(Result)),
?_assertEqual(1, proplists:get_value("cache", Result))
].
read_cgroups_metrics_test_() ->
KeyValueSources = [
"memory.stat",
"unknown.stat"
],
SingleValueSources = [
{limit, "memory.limit_in_bytes"},
{usage, "memory.usage_in_bytes"},
{unknown_metric, "some.unknown_metric"}
],
{setup,
fun() -> ets:new(?MODULE, []) end,
fun(Table) -> ets:delete(Table) end,
fun(Table) ->
ReadResult =
read_cgroups_metrics(
Table, memory, ?TEST_FILE("memory/foo"),
KeyValueSources, SingleValueSources
),
ExpectedErrors = [
{{unknown_metric, "some.unknown_metric"}, enoent},
{"unknown.stat", enoent}
],
ExpectedMetrics =
[{{memory,pgpgout},6},
{{memory,pgfault},7},
{{memory,rss},2},
{{memory,mapped_file},4},
{{memory,pgmajfault},8},
{{memory,pgpgin},5},
{{memory,usage},1},
{{memory,cache},1},
{{memory,limit},2}
],
[
?_assertEqual( lists:sort(ExpectedErrors), lists:sort(ReadResult) ),
?_assertEqual(length(ExpectedMetrics), length(ets:tab2list(Table))),
lists:map(
fun({Key, Value}) ->
?_assertEqual( [{Key, Value}], ets:lookup(Table, Key) )
end, ExpectedMetrics
)
]
end
}.
-endif.

98
src/cg_mon_reader.erl Normal file
View File

@ -0,0 +1,98 @@
-module(cg_mon_reader).
-author("Viacheslav V. Kovalev").
-define(PROVIDED_RESOURCE, memory).
-define(KEY_VALUE_SOURCES, ["memory.stat"]).
-define(SINGLE_VALUE_SOURCES, []).
%% API
-export([
start_link/4,
read_meter/2,
read_meter/3
]).
-export([
init/1,
handle_call/3,
handle_cast/2,
handle_info/2,
code_change/3,
terminate/2
]).
%%%===================================================================
%%% Api functions
%%%===================================================================
start_link(ProvidedResource, KeyValueSources, SingleValueSources, FullCgroupPath) ->
gen_server:start_link(
?MODULE, {ProvidedResource, KeyValueSources, SingleValueSources, FullCgroupPath}, []
).
read_meter(Resource, Meter, DefaultValue) ->
case ets:lookup(cg_mon_app:metrics_table(), {Resource, Meter}) of
[] ->
DefaultValue;
[{_, Value}] ->
Value
end.
read_meter(Resource, Meter) ->
read_meter(Resource, Meter, undefined).
-record(
state, {
provided_resource,
key_value_sources,
single_value_sources,
full_cgroup_path
}
).
init({ProvidedResource, KeyValueSources, SingleValueSources, FullCgroupPath}) ->
error_logger:info_msg(
"Starting cg_mon_reader for resource ~p to read from ~p",
[ProvidedResource, FullCgroupPath]
),
erlang:send( self(), update ),
{ok, #state{
full_cgroup_path = FullCgroupPath,
provided_resource = ProvidedResource,
key_value_sources = KeyValueSources,
single_value_sources = SingleValueSources
}}.
handle_call(_, _, State) ->
{reply, ok, State}.
handle_cast(_, State) ->
{noreply, State}.
handle_info(
update,
#state{
full_cgroup_path = FullCgroupPath,
provided_resource = ProvidedResource,
key_value_sources = KeyValueSources,
single_value_sources = SingleValueSources
} = State
) ->
cg_mon_lib:read_cgroups_metrics(
cg_mon_app:metrics_table(), ProvidedResource, FullCgroupPath,
KeyValueSources, SingleValueSources
),
erlang:send_after( cg_mon_app:update_interval(), self(), update ),
{noreply, State};
handle_info(_, State) ->
{noreply, State}.
code_change(_, State, _) ->
{ok, State}.
terminate(_, _) ->
ok.

76
test/cgmon_SUITE.erl Normal file
View File

@ -0,0 +1,76 @@
-module(cgmon_SUITE).
-author("Viacheslav V. Kovalev").
-include_lib("common_test/include/ct.hrl").
%% API
-export([
all/0,
groups/0,
init_per_group/2,
end_per_group/2
]).
%% Test cases
-export([
successful_start_stop/1,
no_discovery_file/1
]).
all() ->
[
{group, start_stop_tests}
].
groups() ->
[
{start_stop_tests, [sequence], [
successful_start_stop,
no_discovery_file
]},
{cg_mem_sup_tests, [sequence], [
update_memory_metrics
]}
].
init_per_group(start_stop_tests, Config) ->
CgroupRoot = ?config(data_dir, Config),
ok = application:load(cg_mon),
ok = application:set_env( cg_mon, update_interval, 200 ),
[ {cgroup_root, CgroupRoot} | Config ].
end_per_group(start_stop_tests, Config) ->
application:stop(cg_mon),
ok = application:unload(cg_mon),
Config.
successful_start_stop(Config) ->
CgroupRoot = ?config(cgroup_root, Config),
CgroupDiscoveryFile = filename:join(CgroupRoot, "cgroup"),
ok = application:set_env(cg_mon, cgroup_root, CgroupRoot),
ok = application:set_env(cg_mon, cgroup_discovery_file, CgroupDiscoveryFile),
ok = application:start(cg_mon),
timer:sleep(100),
11 = cg_mem_sup:cache(),
12 = cg_mem_sup:rss(),
13 = cg_mem_sup:rss_huge(),
ok = application:stop(cg_mon).
no_discovery_file(Config) ->
CgroupRoot = ?config(cgroup_root, Config),
CgroupDiscoveryFile = filename:join(CgroupRoot, "non_existent_file"),
ok = application:set_env(cg_mon, cgroup_root, CgroupRoot),
ok = application:set_env(cg_mon, cgroup_discovery_file, CgroupDiscoveryFile),
{error, {cgroups_error, _}} = application:start(cg_mon).

View File

@ -0,0 +1,3 @@
1:memory:/foo
2:cpu,cpuacct:/bar
3:cpuset:/

View File

@ -0,0 +1,4 @@
cache 11
rss 12
rss_huge 13
swap 14

View File

@ -0,0 +1,4 @@
cache 1
rss 2
rss_huge 3
swap 4