Mirror/dhtcrawler2
1
0
Fork 0
mirror of https://github.com/btdig/dhtcrawler2.git synced 2025-01-18 20:21:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix hash-writer cache-writting issure

Browse Source
This commit is contained in:
Kevin Lynx 2013-07-18 13:59:37 +08:00
parent cd9ae2ec53
commit 4882cbf692
1 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/crawler/hash_cache_writer.erl
View File

@ -18,6 +18,7 @@
-record(state, {cache_time, cache_max}).
-define(TBLNAME, hash_table).
-define(DBPOOL, hash_write_db).
-define(BATCH_INSERT, 100).
start_link(IP, Port, DBConn, MaxTime, MaxCnt) ->
gen_server:start_link({local, srv_name()}, ?MODULE, [IP, Port, DBConn, MaxTime, MaxCnt], []).
@ -107,13 +108,20 @@ to_docs(Key, ReqAt) ->
do_save(Conn, Docs, ReqSum) ->
?T(?FMT("send ~p hashes req-count ~p to db", [length(Docs), ReqSum])),
save_hashes(Conn, Docs, 1),
db_system:stats_cache_query_inserted(Conn, length(Docs)),
db_system:stats_query_inserted(Conn, ReqSum).
save_hashes(Conn, Docs, Pos) when Pos < length(Docs) ->
SubDocs = lists:sublist(Docs, Pos, ?BATCH_INSERT),
% `safe' may cause this process message queue increasing, but `unsafe' may cause the
% database crashes.
mongo:do(safe, master, Conn, ?HASH_DBNAME, fun() ->
mongo:insert(?HASH_COLLNAME, Docs)
mongo:insert(?HASH_COLLNAME, SubDocs)
end),
db_system:stats_cache_query_inserted(Conn, length(Docs)),
db_system:stats_query_inserted(Conn, ReqSum).
save_hashes(Conn, Docs, Pos + ?BATCH_INSERT);
save_hashes(_, _, _) ->
ok.
get_req_cnt(Hash) ->
case ets:lookup(?TBLNAME, Hash) of