Guest User

Untitled

a guest
Feb 20th, 2018
114
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 6.67 KB | None | 0 0
  1. %% @doc A small LRU list container.
  2. %% @todo Add better documentation.
  3. %% @todo Find edge case bugs in purging.
  4. -module(lrulist).
  5. -author("Nick Gerakines <nick@gerakines.net>").
  6.  
  7. -export([new/0, new/1, get/2, insert/3, remove/2, purge/1, keys/1]).
  8.  
  9. -define(EXPIRE_RULES, [expire, slidingexpire]).
  10.  
  11. -include_lib("eunit/include/eunit.hrl").
  12.  
  13. %% ---
  14. %% Public functions
  15.  
  16. %% @doc Create a new LRU container with the default size (100).
  17. new() ->
  18. new(100).
  19.  
  20. %% @doc Create a new LRU container with a max size.
  21. new(Max) when Max > 0 -> {Max, gb_trees:empty()}.
  22.  
  23. %% @doc Fetch data from a LRU list based on a key.
  24. get(Key, LRUL = {Max, Tree}) ->
  25. case gb_trees:lookup(Key, Tree) of
  26. none -> {none, LRUL};
  27. {value, Data} ->
  28. case expire_rules(?EXPIRE_RULES, Key, Data) of
  29. true ->
  30. NewLRUL = remove(Key, LRUL),
  31. {none, NewLRUL};
  32. false ->
  33. Now = calendar:datetime_to_gregorian_seconds({date(), time()}),
  34. UpdatedTree = gb_trees:enter(Key, [{lastAccess, Now} | Data], Tree),
  35. {{ok, proplists:get_value(value, Data)}, {Max, UpdatedTree}}
  36. end
  37. end.
  38.  
  39. %% This is the same as insert(Key, Value, LRUContainer, []).
  40. insert(Key, Value, {Max, Tree}) ->
  41. insert(Key, Value, {Max, Tree}, []).
  42.  
  43. %% @doc Insert a new value into the container.
  44. %% @todo document options.
  45. insert(Key, Value, {Max, Tree}, Options) ->
  46. PropList = [{value, Value} | Options],
  47. NewTree = gb_trees:enter(Key, PropList, Tree),
  48. {Max, NewTree2} = case [Max > 0, gb_trees:size(Tree) > Max] of
  49. [true, true] -> purge({Max, NewTree});
  50. _ -> {Max, NewTree}
  51. end,
  52. {ok, {Max, NewTree2}}.
  53.  
  54. %% @doc Remove an item from the container.
  55. %% @todo Check for edge cases.
  56. remove(Key, {Max, Tree}) ->
  57. NewTree = gb_trees:delete_any(Key, Tree),
  58. {Max, NewTree}.
  59.  
  60. %% @doc Attempt to purge expired and bloating items from the LRU container.
  61. %% @todo Add Max vs Max * .75 checks
  62. %% @todo Document the Max * .75 rule
  63. purge({Max, Tree}) ->
  64. TreeB = purge_rules(expire, Tree, Max),
  65. NewTree = purge_rules(least_used, TreeB, Max),
  66. BalancedTree = gb_trees:balance(NewTree),
  67. {Max, BalancedTree}.
  68.  
  69. keys({_Max, Tree}) ->
  70. gb_trees:keys(Tree).
  71.  
  72. %% ---
  73. %% Private functions
  74.  
  75. expire_rules([], _Key, _LRU) -> false;
  76. %% Rule 'expire' -- If an 'absoluteExpire' key/value tuple is set as an
  77. %% option when creating the item via insert/4, this rule will remove items
  78. %% that have a hard expiration time.
  79. expire_rules([expire | Rules], Key, Data ) ->
  80. Now = calendar:datetime_to_gregorian_seconds({date(), time()}),
  81. case proplists:get_value(absoluteExpire, Data, Now) < Now of
  82. true -> true;
  83. false -> expire_rules(Rules, Key, Data)
  84. end;
  85. %% Rule 'slidingexpire' -- If a 'slidingExpire' key/value tuple is set as an
  86. %% option when creating the item via insert/4, this rule will remove keys
  87. %% based on a relative lifespan set by that tuple based on when it was last
  88. %% accessed. In other words this expiration rule is used to expire items
  89. %% based on when they are read from the first time.
  90. expire_rules([slidingexpire | Rules], Key, Data) ->
  91. Now = calendar:datetime_to_gregorian_seconds({date(), time()}),
  92. case [proplists:get_value(lastAccess, Data), proplists:get_value(slidingExpire, Data)] of
  93. [undefined, _] -> expire_rules(Rules, Key, Data);
  94. [_, undefined] -> expire_rules(Rules, Key, Data);
  95. [LastAccess, SlidingExpire] ->
  96. case LastAccess + (SlidingExpire * 1000) < Now of
  97. true -> true;
  98. false -> expire_rules(Rules, Key, Data)
  99. end
  100. end.
  101.  
  102. %% Purge rule 'expire' -- This is the purge_* equiv of the 'expire' expire
  103. %% rule above.
  104. purge_rules(expire, Tree, _Max) ->
  105. Iter = gb_trees:iterator(Tree),
  106. Keys = expire_iter(gb_trees:next(Iter), []),
  107. lists:foldl(
  108. fun(Key, TmpTree) ->
  109. gb_trees:delete_any(Key, TmpTree)
  110. end,
  111. Tree,
  112. Keys
  113. );
  114.  
  115. %% Purge rule 'least_used' -- This purge rule will attempt to sort all items
  116. %% by the last time they were accessed. Once it has the sorted list it will
  117. %% attempt to cut the list down to 75% of Max and returns the new list.
  118. %% NOTE: When a container is more write active than read active the pruge
  119. %% process will be called alot. To limit this as much as possible when we
  120. %% purge, we get rid of more than we need to.
  121. purge_rules(least_used, Tree, Max) ->
  122. List = gb_trees:to_list(Tree),
  123. SortedList = lists:sort(fun priority_sort/2, List),
  124. RemoveList = lists:nthtail(round(Max * 0.75), SortedList),
  125. lists:foldl(
  126. fun(Key, TmpTree) -> gb_trees:delete_any(Key, TmpTree) end,
  127. Tree,
  128. [Key || {Key, _} <- RemoveList]
  129. ).
  130.  
  131. %% This funciton takes advantage of the gb_trees:iterator/1 and
  132. %% gb_trees:next/1 functions to quickly iterate through a tree without
  133. %% having to do heavy break-down and build-up operations on the internal
  134. %% tree.
  135. expire_iter(none, Acc) -> Acc;
  136. expire_iter({Key, Value, Iter}, Acc) ->
  137. Now = calendar:datetime_to_gregorian_seconds({date(), time()}),
  138. NewAcc = case proplists:get_value(absoluteExpire, Value, Now) < Now of
  139. true -> [Key | Acc];
  140. false -> Acc
  141. end,
  142. expire_iter(gb_trees:next(Iter), NewAcc).
  143.  
  144. %% The sort function used in the 'least_used' purge rule.
  145. priority_sort({_, A}, {_, B}) ->
  146. proplists:get_value(lastAccess, A) > proplists:get_value(lastAccess, B).
  147.  
  148. %% ---
  149. %% Test Functions
  150. %% call with `lrulist:test().`
  151.  
  152. %% basic_test_ -- Do some basic writes and reads
  153. basic_test_() ->
  154. {
  155. "Basic setting and getting.",
  156. fun() ->
  157. L1 = lrulist:new(),
  158. {ok, L2} = lrulist:insert("starbucks", 4, L1),
  159. {ok, L3} = lrulist:insert("petes", 2, L2),
  160. {ok, L4} = lrulist:insert("distel", none, L3),
  161. {{ok, 4}, L5} = lrulist:get("starbucks", L4),
  162. {{ok, 2}, L6} = lrulist:get("petes", L5),
  163. {{ok, none}, _L7} = lrulist:get("distel", L6)
  164. end
  165. }.
  166.  
  167. %% purge_test_ -- Do some writes and reads while tripping the Max of a lru
  168. %% container.
  169. purge_test_() ->
  170. fun() ->
  171. LRUList = lists:foldl(
  172. fun(User, Tmplru) ->
  173. {ok, Tmplru2} = lrulist:insert(User, User, Tmplru),
  174. {{ok, User}, Tmplru3} = lrulist:get(User, Tmplru2),
  175. Tmplru3
  176. end,
  177. lrulist:new(15),
  178. [lists:concat(["user", X]) || X <- lists:seq(1, 20)]
  179. ),
  180. [lists:concat(["user", X]) || X <- lists:seq(1, 15)] == lrulist:keys(LRUList)
  181. end.
Add Comment
Please, Sign In to add comment