refactor progress
This commit is contained in:
Hampus Kraft
@@ -17,9 +17,7 @@
|
||||
|
||||
-module(guild_client).
|
||||
|
||||
-export([
|
||||
voice_state_update/3
|
||||
]).
|
||||
-export([voice_state_update/3]).
|
||||
|
||||
-export_type([
|
||||
voice_state_update_success/0,
|
||||
@@ -27,7 +25,10 @@
|
||||
voice_state_update_result/0
|
||||
]).
|
||||
|
||||
-define(DEFAULT_TIMEOUT, 12000).
|
||||
-define(CIRCUIT_BREAKER_TABLE, guild_circuit_breaker).
|
||||
-define(FAILURE_THRESHOLD, 5).
|
||||
-define(RECOVERY_TIMEOUT_MS, 30000).
|
||||
-define(MAX_CONCURRENT, 50).
|
||||
|
||||
-type voice_state_update_success() :: #{
|
||||
success := true,
|
||||
@@ -44,10 +45,39 @@
|
||||
{ok, voice_state_update_success()}
|
||||
| {error, timeout}
|
||||
| {error, noproc}
|
||||
| {error, circuit_breaker_open}
|
||||
| {error, too_many_requests}
|
||||
| {error, atom(), atom()}.
|
||||
|
||||
-type circuit_state() :: closed | open | half_open.
|
||||
|
||||
-spec voice_state_update(pid(), map(), timeout()) -> voice_state_update_result().
|
||||
voice_state_update(GuildPid, Request, Timeout) ->
|
||||
ensure_table(),
|
||||
case acquire_slot(GuildPid) of
|
||||
ok ->
|
||||
try
|
||||
execute_with_circuit_breaker(GuildPid, Request, Timeout)
|
||||
after
|
||||
release_slot(GuildPid)
|
||||
end;
|
||||
{error, Reason} ->
|
||||
{error, Reason}
|
||||
end.
|
||||
|
||||
-spec execute_with_circuit_breaker(pid(), map(), timeout()) -> voice_state_update_result().
|
||||
execute_with_circuit_breaker(GuildPid, Request, Timeout) ->
|
||||
case get_circuit_state(GuildPid) of
|
||||
open ->
|
||||
{error, circuit_breaker_open};
|
||||
State when State =:= closed; State =:= half_open ->
|
||||
Result = do_call(GuildPid, Request, Timeout),
|
||||
update_circuit_state(GuildPid, Result, State),
|
||||
Result
|
||||
end.
|
||||
|
||||
-spec do_call(pid(), map(), timeout()) -> voice_state_update_result().
|
||||
do_call(GuildPid, Request, Timeout) ->
|
||||
try gen_server:call(GuildPid, {voice_state_update, Request}, Timeout) of
|
||||
Response when is_map(Response) ->
|
||||
case maps:get(success, Response, false) of
|
||||
@@ -57,12 +87,138 @@ voice_state_update(GuildPid, Request, Timeout) ->
|
||||
{error, Category, ErrorAtom} when is_atom(Category), is_atom(ErrorAtom) ->
|
||||
{error, Category, ErrorAtom}
|
||||
catch
|
||||
exit:{timeout, _} ->
|
||||
{error, timeout};
|
||||
exit:{noproc, _} ->
|
||||
{error, noproc};
|
||||
exit:{normal, _} ->
|
||||
{error, noproc}
|
||||
exit:{timeout, _} -> {error, timeout};
|
||||
exit:{noproc, _} -> {error, noproc};
|
||||
exit:{normal, _} -> {error, noproc}
|
||||
end.
|
||||
|
||||
-spec get_circuit_state(pid()) -> circuit_state().
|
||||
get_circuit_state(GuildPid) ->
|
||||
case safe_lookup(GuildPid) of
|
||||
[] ->
|
||||
closed;
|
||||
[{_, #{state := open, opened_at := OpenedAt}}] ->
|
||||
Now = erlang:system_time(millisecond),
|
||||
case Now - OpenedAt > ?RECOVERY_TIMEOUT_MS of
|
||||
true -> half_open;
|
||||
false -> open
|
||||
end;
|
||||
[{_, #{state := State}}] ->
|
||||
State
|
||||
end.
|
||||
|
||||
-spec update_circuit_state(pid(), voice_state_update_result(), circuit_state()) -> ok.
|
||||
update_circuit_state(GuildPid, Result, PrevState) ->
|
||||
IsSuccess = is_success_result(Result),
|
||||
case {IsSuccess, PrevState} of
|
||||
{true, half_open} ->
|
||||
ets:delete(?CIRCUIT_BREAKER_TABLE, GuildPid),
|
||||
ok;
|
||||
{true, closed} ->
|
||||
reset_failures(GuildPid);
|
||||
{false, _} ->
|
||||
record_failure(GuildPid)
|
||||
end.
|
||||
|
||||
-spec is_success_result(voice_state_update_result()) -> boolean().
|
||||
is_success_result({ok, _}) -> true;
|
||||
is_success_result(_) -> false.
|
||||
|
||||
-spec reset_failures(pid()) -> ok.
|
||||
reset_failures(GuildPid) ->
|
||||
case safe_lookup(GuildPid) of
|
||||
[{_, State}] ->
|
||||
ets:insert(?CIRCUIT_BREAKER_TABLE, {GuildPid, State#{failures => 0}}),
|
||||
ok;
|
||||
[] ->
|
||||
ok
|
||||
end.
|
||||
|
||||
-spec record_failure(pid()) -> ok.
|
||||
record_failure(GuildPid) ->
|
||||
Now = erlang:system_time(millisecond),
|
||||
case safe_lookup(GuildPid) of
|
||||
[] ->
|
||||
ets:insert(
|
||||
?CIRCUIT_BREAKER_TABLE,
|
||||
{GuildPid, #{
|
||||
state => closed,
|
||||
failures => 1,
|
||||
concurrent => 0
|
||||
}}
|
||||
),
|
||||
ok;
|
||||
[{_, #{failures := F} = State}] when F + 1 >= ?FAILURE_THRESHOLD ->
|
||||
ets:insert(
|
||||
?CIRCUIT_BREAKER_TABLE,
|
||||
{GuildPid, State#{
|
||||
state => open,
|
||||
failures => F + 1,
|
||||
opened_at => Now
|
||||
}}
|
||||
),
|
||||
ok;
|
||||
[{_, #{failures := F} = State}] ->
|
||||
ets:insert(?CIRCUIT_BREAKER_TABLE, {GuildPid, State#{failures => F + 1}}),
|
||||
ok
|
||||
end.
|
||||
|
||||
-spec acquire_slot(pid()) -> ok | {error, too_many_requests}.
|
||||
acquire_slot(GuildPid) ->
|
||||
case safe_lookup(GuildPid) of
|
||||
[] ->
|
||||
ets:insert(
|
||||
?CIRCUIT_BREAKER_TABLE,
|
||||
{GuildPid, #{
|
||||
state => closed,
|
||||
failures => 0,
|
||||
concurrent => 1
|
||||
}}
|
||||
),
|
||||
ok;
|
||||
[{_, #{concurrent := C}}] when C >= ?MAX_CONCURRENT ->
|
||||
{error, too_many_requests};
|
||||
[{_, #{concurrent := C} = State}] ->
|
||||
ets:insert(?CIRCUIT_BREAKER_TABLE, {GuildPid, State#{concurrent => C + 1}}),
|
||||
ok
|
||||
end.
|
||||
|
||||
-spec release_slot(pid()) -> ok.
|
||||
release_slot(GuildPid) ->
|
||||
case safe_lookup(GuildPid) of
|
||||
[{_, #{concurrent := C} = State}] when C > 0 ->
|
||||
ets:insert(?CIRCUIT_BREAKER_TABLE, {GuildPid, State#{concurrent => C - 1}}),
|
||||
ok;
|
||||
_ ->
|
||||
ok
|
||||
end.
|
||||
|
||||
-spec safe_lookup(pid()) -> list().
|
||||
safe_lookup(GuildPid) ->
|
||||
try ets:lookup(?CIRCUIT_BREAKER_TABLE, GuildPid) of
|
||||
Result -> Result
|
||||
catch
|
||||
error:badarg -> []
|
||||
end.
|
||||
|
||||
-spec ensure_table() -> ok.
|
||||
ensure_table() ->
|
||||
case ets:whereis(?CIRCUIT_BREAKER_TABLE) of
|
||||
undefined ->
|
||||
try
|
||||
ets:new(?CIRCUIT_BREAKER_TABLE, [
|
||||
named_table,
|
||||
public,
|
||||
set,
|
||||
{read_concurrency, true},
|
||||
{write_concurrency, true}
|
||||
]),
|
||||
ok
|
||||
catch
|
||||
error:badarg -> ok
|
||||
end;
|
||||
_ ->
|
||||
ok
|
||||
end.
|
||||
|
||||
-ifdef(TEST).
|
||||
@@ -72,4 +228,136 @@ module_exports_test() ->
|
||||
Exports = guild_client:module_info(exports),
|
||||
?assert(lists:member({voice_state_update, 3}, Exports)).
|
||||
|
||||
ensure_table_creates_table_test() ->
|
||||
catch ets:delete(?CIRCUIT_BREAKER_TABLE),
|
||||
?assertEqual(undefined, ets:whereis(?CIRCUIT_BREAKER_TABLE)),
|
||||
ensure_table(),
|
||||
?assertNotEqual(undefined, ets:whereis(?CIRCUIT_BREAKER_TABLE)).
|
||||
|
||||
ensure_table_idempotent_test() ->
|
||||
ensure_table(),
|
||||
ensure_table(),
|
||||
?assertNotEqual(undefined, ets:whereis(?CIRCUIT_BREAKER_TABLE)).
|
||||
|
||||
acquire_slot_creates_entry_test() ->
|
||||
ensure_table(),
|
||||
Pid = spawn(fun() ->
|
||||
receive
|
||||
done -> ok
|
||||
end
|
||||
end),
|
||||
ets:delete_all_objects(?CIRCUIT_BREAKER_TABLE),
|
||||
?assertEqual(ok, acquire_slot(Pid)),
|
||||
[{Pid, State}] = ets:lookup(?CIRCUIT_BREAKER_TABLE, Pid),
|
||||
?assertEqual(1, maps:get(concurrent, State)),
|
||||
Pid ! done.
|
||||
|
||||
acquire_slot_increments_test() ->
|
||||
ensure_table(),
|
||||
Pid = spawn(fun() ->
|
||||
receive
|
||||
done -> ok
|
||||
end
|
||||
end),
|
||||
ets:delete_all_objects(?CIRCUIT_BREAKER_TABLE),
|
||||
acquire_slot(Pid),
|
||||
acquire_slot(Pid),
|
||||
[{Pid, State}] = ets:lookup(?CIRCUIT_BREAKER_TABLE, Pid),
|
||||
?assertEqual(2, maps:get(concurrent, State)),
|
||||
Pid ! done.
|
||||
|
||||
release_slot_decrements_test() ->
|
||||
ensure_table(),
|
||||
Pid = spawn(fun() ->
|
||||
receive
|
||||
done -> ok
|
||||
end
|
||||
end),
|
||||
ets:delete_all_objects(?CIRCUIT_BREAKER_TABLE),
|
||||
acquire_slot(Pid),
|
||||
acquire_slot(Pid),
|
||||
release_slot(Pid),
|
||||
[{Pid, State}] = ets:lookup(?CIRCUIT_BREAKER_TABLE, Pid),
|
||||
?assertEqual(1, maps:get(concurrent, State)),
|
||||
Pid ! done.
|
||||
|
||||
get_circuit_state_closed_test() ->
|
||||
ensure_table(),
|
||||
Pid = spawn(fun() ->
|
||||
receive
|
||||
done -> ok
|
||||
end
|
||||
end),
|
||||
ets:delete_all_objects(?CIRCUIT_BREAKER_TABLE),
|
||||
?assertEqual(closed, get_circuit_state(Pid)),
|
||||
Pid ! done.
|
||||
|
||||
get_circuit_state_open_test() ->
|
||||
ensure_table(),
|
||||
Pid = spawn(fun() ->
|
||||
receive
|
||||
done -> ok
|
||||
end
|
||||
end),
|
||||
ets:delete_all_objects(?CIRCUIT_BREAKER_TABLE),
|
||||
Now = erlang:system_time(millisecond),
|
||||
ets:insert(
|
||||
?CIRCUIT_BREAKER_TABLE,
|
||||
{Pid, #{
|
||||
state => open,
|
||||
failures => 5,
|
||||
concurrent => 0,
|
||||
opened_at => Now
|
||||
}}
|
||||
),
|
||||
?assertEqual(open, get_circuit_state(Pid)),
|
||||
Pid ! done.
|
||||
|
||||
get_circuit_state_half_open_test() ->
|
||||
ensure_table(),
|
||||
Pid = spawn(fun() ->
|
||||
receive
|
||||
done -> ok
|
||||
end
|
||||
end),
|
||||
ets:delete_all_objects(?CIRCUIT_BREAKER_TABLE),
|
||||
OldTime = erlang:system_time(millisecond) - ?RECOVERY_TIMEOUT_MS - 1000,
|
||||
ets:insert(
|
||||
?CIRCUIT_BREAKER_TABLE,
|
||||
{Pid, #{
|
||||
state => open,
|
||||
failures => 5,
|
||||
concurrent => 0,
|
||||
opened_at => OldTime
|
||||
}}
|
||||
),
|
||||
?assertEqual(half_open, get_circuit_state(Pid)),
|
||||
Pid ! done.
|
||||
|
||||
record_failure_opens_circuit_test() ->
|
||||
ensure_table(),
|
||||
Pid = spawn(fun() ->
|
||||
receive
|
||||
done -> ok
|
||||
end
|
||||
end),
|
||||
ets:delete_all_objects(?CIRCUIT_BREAKER_TABLE),
|
||||
ets:insert(
|
||||
?CIRCUIT_BREAKER_TABLE,
|
||||
{Pid, #{
|
||||
state => closed,
|
||||
failures => ?FAILURE_THRESHOLD - 1,
|
||||
concurrent => 0
|
||||
}}
|
||||
),
|
||||
record_failure(Pid),
|
||||
[{Pid, State}] = ets:lookup(?CIRCUIT_BREAKER_TABLE, Pid),
|
||||
?assertEqual(open, maps:get(state, State)),
|
||||
Pid ! done.
|
||||
|
||||
is_success_result_test() ->
|
||||
?assertEqual(true, is_success_result({ok, #{}})),
|
||||
?assertEqual(false, is_success_result({error, timeout})),
|
||||
?assertEqual(false, is_success_result({error, noproc})).
|
||||
|
||||
-endif.
|
||||
|
||||
Reference in New Issue
Block a user