SHARE
TWEET

abhinav

a guest Jan 18th, 2011 140 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. -module(markov_distributed).
  2. -compile(export_all).
  3. -author({jha, abhinav}).
  4. -define(INITPOINT, 50).
  5. -define(ARITY, 3).
  6.  
  7. start() -> start("corpus.txt").
  8. start(Corpus) ->
  9.     {ok, Bin} = file:read_file(Corpus),
  10.     Words = string:tokens(binary_to_list(Bin), "\n "),
  11.     Tab = ets:new(words, [bag]),
  12.     form_dictionary(Words, Tab, ?ARITY),
  13.     Start = mrandom(?INITPOINT),
  14.     State0 = lists:sublist(Words, Start, ?ARITY - 1),
  15.     Sentence = spew(Tab, State0, []),
  16.     ets:delete(Tab),
  17.     io:format("~p~n", [string:join(Sentence, " ")]).
  18.  
  19. spew(Tab, [_|L]=State, Acc)->
  20.     % End at a word that ends with a fullstop, once we're greater than 20 words.
  21.     case (length(Acc) >= 15) and (string:rchr(lists:last(L), $.) =:= length(lists:last(L))) of
  22.         true -> lists:reverse(Acc);
  23.         false ->
  24.             Objects = ets:lookup(Tab, list_to_tuple(State)),
  25.             {_,Snew} = lists:nth(mrandom(length(Objects)), Objects),
  26.             spew(Tab, L ++ [Snew], [Snew | Acc])
  27.     end.
  28.  
  29. mrandom(Max) ->
  30.     {A, B, C} = now(),
  31.     random:seed(A,B,C),
  32.     random:uniform(Max).
  33.  
  34. form_dictionary([], _, _) -> void;
  35. form_dictionary([H|T], Tab, N)->
  36.     Self = self(),
  37.     Pids = [spawn(fun()->ngrams(lists:nthtail(Y-1, [H|T]), N, [], Self) end) || Y<- lists:seq(1, length([H|T]))],
  38.     gather(Tab, length(Pids)).
  39.  
  40. gather(_, 0) -> void;
  41. gather(Tab, X)->
  42.     receive
  43.         {ok, Pid, Ngram} ->
  44.             ets:insert(Tab, Ngram),
  45.             %io:format("Received ~p from ~p. X = ~p ~n", [Ngram, Pid, X]),
  46.             gather(Tab, X-1);
  47.         _-> gather(Tab, X-1)
  48.     end.
  49.  
  50.  
  51. ngrams([], _, _, Self) -> Self!void;
  52. ngrams([H|_], 1, Acc, Self)->
  53.     %io:format("~p: ~p~n", [self(), {list_to_tuple(lists:reverse(Acc)), H}]),
  54.     Self ! {ok, self(), {list_to_tuple(lists:reverse(Acc)), H}};
  55. ngrams([H|T], X, Acc, Self)-> ngrams(T, X-1, [H|Acc], Self).
RAW Paste Data
Top