Advertisement
Guest User

Untitled

a guest
Oct 27th, 2011
128
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Erlang 2.59 KB | None | 0 0
  1. -module(galdl).
  2. -export([main/1]).
  3.  
  4. dlurl() ->
  5.     receive
  6.     {job, [Url, Num, PPID]} ->
  7.         RealUrl = try lists:flatten(io_lib:format(Url, [Num]))
  8.               catch
  9.               error:_ ->
  10.                   io:format("An error occured. Check your URL parameter.~n"),
  11.                   halt(1)
  12.               end,
  13.         io:format("Downloading ~ts~n", [RealUrl]),
  14.         Basename = filename:basename(RealUrl),
  15.         case httpc:request(get, {RealUrl, []}, [], [{stream, Basename}]) of
  16.         {ok, saved_to_file} ->
  17.             io:format("Saved file ~ts~n", [Basename]),
  18.             PPID ! {done, {self(), Num}};
  19.         {ok, {{_HTTP, Code, _Msg}, _Headers, _Body}} when Code > 499 ->
  20.             PPID ! {temperror, {self(), Num}};
  21.         {ok, {{_HTTP, Code, _Msg}, _Headers, _Body}} when Code > 399, Code < 500 ->
  22.             PPID ! {permerror, {self(), Num}};
  23.         {error, Error} ->
  24.             io:format("A network error occured: ~p~n", [Error]),
  25.             PPID ! {temperror, {self(), Num}}
  26.         end
  27.     end,
  28.     dlurl().
  29.  
  30. loop(Url, From, To, TLimit, Proclist, Busylist) ->
  31.     if
  32.     length(Busylist) == 0, From > To ->
  33.         done;
  34.     length(Busylist) < TLimit, From =< To ->
  35.         PID = lists:nth(1, Proclist -- Busylist),
  36.         io:format("Setting the task #~w for the process ~w, Busylist: ~w~n", [From, PID, Busylist]),
  37.         PID ! {job, [Url, From, self()]},
  38.         loop(Url, From + 1, To, TLimit, Proclist, Busylist ++ [PID]);
  39.     true ->
  40.         NewBusylist = receive
  41.                   {done, {PID, Num}} ->
  42.                   io:format("Thread ~w finished #~w~n", [PID, Num]),
  43.                   Busylist -- [PID];
  44.                   {permerror, {PID, Num}} ->
  45.                   io:format("Thread ~w got a permanent error retrieving #~w, skipping~n", [PID, Num]),
  46.                   Busylist -- [PID];
  47.                   {temperror, {PID, Num}} ->
  48.                   io:format("Thread ~w haven't got ~w, retrying~n", [PID, Num]),
  49.                   PID ! {job, [Url, Num, self()]},
  50.                   Busylist
  51.               end,
  52.         loop(Url, From, To, TLimit, Proclist, NewBusylist)
  53.     end.
  54.            
  55. main([Url, From, To, TLimit]) ->
  56.     inets:start(),
  57.     %%inets:start(httpc, [{profile, myprofile}]),
  58.     %%httpc:set_options([{verbose, debug}]),
  59.     io:format("Module name: ~w PID: ~w~n", [?MODULE, self()]),
  60.     FromI = list_to_integer(From),
  61.     ToI = list_to_integer(To),
  62.     TLimitI = list_to_integer(TLimit),
  63.     loop(Url, FromI, ToI, TLimitI, [spawn(fun() -> dlurl() end) || _ <- lists:seq(1, TLimitI)], []),
  64.     io:format("Done!~n"),
  65.     halt(0);
  66.  
  67. main([Url, From, To]) ->
  68.     main([Url, From, To, "5"]);
  69.  
  70. main(_) ->
  71.     io:format("Usage: <url> <from> <to> [process_count]~nurl should be formatted like http://example.net/path/to/gallery/~~3..0w.jpg for files like 001.jpg, 002.jpg etc.~n"),
  72.     halt(0).
  73.  
  74.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement