Mirror/dhtcrawler2
1
0
Fork 0
mirror of https://github.com/btdig/dhtcrawler2.git synced 2025-01-19 04:31:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

config max download task per hash-reader

Browse Source
This commit is contained in:
Kevin Lynx 2013-07-18 22:04:57 +08:00
parent 4882cbf692
commit 76542be37a
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/hash_reader/hash_reader.erl
View File

@ -19,9 +19,8 @@
-include("db_common.hrl"). -include("db_common.hrl").
% if there's no hash, wait some time % if there's no hash, wait some time
-define(WAIT_TIME, 1*60*1000). -define(WAIT_TIME, 1*60*1000).
% the max concurrent download tasks
-define(MAX_DOWNLOAD, 50).
-define(DOWNLOAD_INTERVAL, 100). -define(DOWNLOAD_INTERVAL, 100).
-define(DEF_MAX_DOWNLOAD, 50).
start_link(DBPool) -> start_link(DBPool) ->
gen_server:start_link(?MODULE, [DBPool], []). gen_server:start_link(?MODULE, [DBPool], []).
@ -79,7 +78,8 @@ handle_info(timeout, State) ->
% when there's no hash to process % when there's no hash to process
handle_info(process_download_hash, State) -> handle_info(process_download_hash, State) ->
#state{downloading = D} = State, #state{downloading = D} = State,
NewD = case D >= ?MAX_DOWNLOAD of MaxD = config:get(max_download_per_reader, ?DEF_MAX_DOWNLOAD),
NewD = case D >= MaxD of
true -> true ->
% the only thing we can do is just wait % the only thing we can do is just wait
timer:send_after(?WAIT_TIME, timeout), timer:send_after(?WAIT_TIME, timeout),
@ -140,7 +140,8 @@ get_req_cnt(Doc) ->
try_download(State, Hash, Doc) -> try_download(State, Hash, Doc) ->
#state{downloading = D} = State, #state{downloading = D} = State,
NewDownloading = case D >= ?MAX_DOWNLOAD of MaxD = config:get(max_download_per_reader, ?DEF_MAX_DOWNLOAD),
NewDownloading = case D >= MaxD of
true -> % put it into the download queue true -> % put it into the download queue
?T(?FMT("reach the max download, insert it to wait queue ~s", [Hash])), ?T(?FMT("reach the max download, insert it to wait queue ~s", [Hash])),
Conn = db_conn(State), Conn = db_conn(State),

1
src/hash_reader/hash_reader_sup.erl
View File

@ -75,4 +75,5 @@ config_default() ->
{load_from_db, false}, {load_from_db, false},
{text_seg, simple}, {text_seg, simple},
{check_cache, false}, {check_cache, false},
{max_download_per_reader, 100},
{torrent_path, "torrents/"}]. {torrent_path, "torrents/"}].