Mirror/dhtcrawler2
1
0
Fork 0
mirror of https://github.com/btdig/dhtcrawler2.git synced 2025-02-27 23:59:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

add some debug log to sphinx_builder, test sphinx_builder when there's no hashes there and got new hashes

Browse Source
This commit is contained in:
Kevin Lynx 2013-08-05 21:31:16 +08:00
parent 7af23fcc49
commit 377912982d
2 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/sphinx_builder/sphinx_builder.erl
View File

@ -89,7 +89,6 @@ worker_run() ->
worker_run(). worker_run().
do_process({_, wait}) -> do_process({_, wait}) ->
?T(?FMT("worker ~p sleep ~p ms", [self(), ?WORKER_WAIT])),
timer:sleep(?WORKER_WAIT); timer:sleep(?WORKER_WAIT);
do_process({ID, Doc}) -> do_process({ID, Doc}) ->
case db_store_mongo:decode_torrent_item(Doc) of case db_store_mongo:decode_torrent_item(Doc) of

8
src/sphinx_builder/sphinx_torrent.erl
View File

@ -13,6 +13,7 @@
terminate/2, terminate/2,
code_change/3]). code_change/3]).
-export([start_link/3, get/0, offset_date/0, try_times/0]). -export([start_link/3, get/0, offset_date/0, try_times/0]).
-import(time_util, [date_time_string/1]).
-define(DBNAME, torrents). -define(DBNAME, torrents).
-define(COLLNAME, hashes). -define(COLLNAME, hashes).
-define(POOLNAME, db_pool). -define(POOLNAME, db_pool).
@ -42,7 +43,7 @@ init([IP, Port, Date]) ->
Conn = mongo_pool:get(?POOLNAME), Conn = mongo_pool:get(?POOLNAME),
db_store_mongo:ensure_date_index(Conn), db_store_mongo:ensure_date_index(Conn),
CheckDate = find_first_date(Date), CheckDate = find_first_date(Date),
io:format("load torrent from ~s~n", [time_util:date_time_string(CheckDate)]), io:format("load torrent from ~s~n", [date_time_string(CheckDate)]),
{ok, #state{date = CheckDate, max = Max}, 0}. {ok, #state{date = CheckDate, max = Max}, 0}.
handle_cast(stop, State) -> handle_cast(stop, State) ->
@ -52,14 +53,18 @@ handle_info(try_load, State) ->
#state{date = Date, tors = Tors, max = Max, try_times = Try} = State, #state{date = Date, tors = Tors, max = Max, try_times = Try} = State,
case length(Tors) < Max of case length(Tors) < Max of
true -> true ->
?T(?FMT("try load more torrents ~p -> ~p", [length(Tors), Max])),
case do_load_torrents(Date, Max) of case do_load_torrents(Date, Max) of
[] -> [] ->
?T(?FMT("load 0 torrent from ~s", [date_time_string(Date)])),
NewDate = forward_date(Date), NewDate = forward_date(Date),
NewTry = case NewDate == Date of NewTry = case NewDate == Date of
true -> true ->
?T(?FMT("no more torrents, wait, try ~p", [Try])),
timer:send_after(?WAIT_TIME, try_load), timer:send_after(?WAIT_TIME, try_load),
Try + 1; Try + 1;
false -> false ->
?T(?FMT("forward date to ~s", [date_time_string(NewDate)])),
timer:send_after(100, try_load), timer:send_after(100, try_load),
Try Try
end, end,
@ -67,6 +72,7 @@ handle_info(try_load, State) ->
Ret -> Ret ->
timer:send_after(100, try_load), timer:send_after(100, try_load),
NewDate = query_created_at(lists:last(Ret)), NewDate = query_created_at(lists:last(Ret)),
?T(?FMT("load ~p torrents, new date ~s", [length(Ret), date_time_string(NewDate)])),
{noreply, State#state{date = NewDate, tors = Tors ++ Ret, try_times = 0}} {noreply, State#state{date = NewDate, tors = Tors ++ Ret, try_times = 0}}
end; end;
false -> false ->