Mirror/dhtcrawler2
1
0
Fork 0
mirror of https://github.com/btdig/dhtcrawler2.git synced 2025-02-23 05:39:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix sphinx builder query range bug; improve sphinx builder db query performance, to avoid `getmore' command and the 4M useless response

Browse Source
This commit is contained in:
Kevin Lynx 2013-08-07 21:50:06 +08:00
parent f887208cd5
commit 4a9b85c973
2 changed files with 27 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/db_store_mongo.erl
View File

@ -71,7 +71,7 @@ search(Conn, Key) when is_list(Key) ->
search_announce_top(Conn, Count) ->
Sel = {'$query', {}, '$orderby', {announce, -1}},
List = mongo_do_slave(Conn, fun() ->
% mongodb-erlang does not provide cursor.limit()/sort() functions, wired
% mongodb-erlang does not provide cursor.limit()/sort() functions, weird
% but it work here
Cursor = mongo:find(?COLLNAME, Sel, [], 0, Count),
mongo_cursor:rest(Cursor)
@ -343,3 +343,27 @@ test_compile() ->
io:format("sphins disabled~n", []).
-endif.
% about mongodb `batchsize' and `getmore':http://blog.nosqlfan.com/html/3996.html
% looks like the `batchsize' argument will take effect on `getmore' command
% 1. load `Batch' docs first, then take them all
% Cursor = mongo:find(?COLLNAME, {}, [], 0, Batch),
% mongo_cursor:take(Cursor, Batch)
% 2. default load 101 docs first
% Cursor = mongo:find(?COLLNAME, {}, [], 0, 0),
% mongo_cursor:take(Cursor, Batch)
% 3. default load 101 docs first, then `getmore' to load 4M data
% Cursor = mongo:find(?COLLNAME, {}, [], 0, 0),
% mongo_cursor:take(Cursor, 102)
% 4. load a batch of docs, and mongodb close the connection
% Cursor = mongo:find(?COLLNAME, {}, [], 0, -Cnt),
% mongo_cursor:rest(Cursor)
test_batch(Cnt) ->
Conn = mongo_pool:get(db_pool),
mongo:do(safe, master, Conn, ?DBNAME, fun() ->
%Cursor = mongo:find(?COLLNAME, {}, [], 0, Cnt),
Cursor = mongo:find(?COLLNAME, {}, [], 0, -Cnt),
%mongo_cursor:take(Cursor, Cnt)
%mongo_cursor:take(Cursor, Cnt)
mongo_cursor:rest(Cursor)
end).

4
src/sphinx_builder/sphinx_torrent.erl
View File

@ -112,7 +112,7 @@ forward_date(Date) ->
% will cause lots of queries
do_load_torrents(Date, Size) ->
Q = {created_at, {'$gt', Date, '$lt', Date + ?DATE_RANGE}},
Q = {created_at, {'$gt', Date, '$lte', Date + ?DATE_RANGE}},
Conn = mongo_pool:get(?POOLNAME),
mongo:do(safe, master, Conn, ?DBNAME, fun() ->
% 1: cost lots of memory even close the cursor
@ -122,7 +122,7 @@ do_load_torrents(Date, Size) ->
%Cursor = mongo:find(?COLLNAME, {}, {}, Skip),
%mongo_cursor:take(Cursor, Size),
% 3:
Cursor = mongo:find(?COLLNAME, Q, {}),
Cursor = mongo:find(?COLLNAME, Q, {}, 0, Size),
Ret = mongo_cursor:take(Cursor, Size),
mongo_cursor:close(Cursor),
Ret