diff --git a/src/db_store_mongo.erl b/src/db_store_mongo.erl index b85d370..1180f87 100644 --- a/src/db_store_mongo.erl +++ b/src/db_store_mongo.erl @@ -71,7 +71,7 @@ search(Conn, Key) when is_list(Key) -> search_announce_top(Conn, Count) -> Sel = {'$query', {}, '$orderby', {announce, -1}}, List = mongo_do_slave(Conn, fun() -> - % mongodb-erlang does not provide cursor.limit()/sort() functions, wired + % mongodb-erlang does not provide cursor.limit()/sort() functions, weird % but it work here Cursor = mongo:find(?COLLNAME, Sel, [], 0, Count), mongo_cursor:rest(Cursor) @@ -343,3 +343,27 @@ test_compile() -> io:format("sphins disabled~n", []). -endif. +% about mongodb `batchsize' and `getmore':http://blog.nosqlfan.com/html/3996.html +% looks like the `batchsize' argument will take effect on `getmore' command +% 1. load `Batch' docs first, then take them all +% Cursor = mongo:find(?COLLNAME, {}, [], 0, Batch), +% mongo_cursor:take(Cursor, Batch) +% 2. default load 101 docs first +% Cursor = mongo:find(?COLLNAME, {}, [], 0, 0), +% mongo_cursor:take(Cursor, Batch) +% 3. default load 101 docs first, then `getmore' to load 4M data +% Cursor = mongo:find(?COLLNAME, {}, [], 0, 0), +% mongo_cursor:take(Cursor, 102) +% 4. load a batch of docs, and mongodb close the connection +% Cursor = mongo:find(?COLLNAME, {}, [], 0, -Cnt), +% mongo_cursor:rest(Cursor) +test_batch(Cnt) -> + Conn = mongo_pool:get(db_pool), + mongo:do(safe, master, Conn, ?DBNAME, fun() -> + %Cursor = mongo:find(?COLLNAME, {}, [], 0, Cnt), + Cursor = mongo:find(?COLLNAME, {}, [], 0, -Cnt), + %mongo_cursor:take(Cursor, Cnt) + %mongo_cursor:take(Cursor, Cnt) + mongo_cursor:rest(Cursor) + end). + diff --git a/src/sphinx_builder/sphinx_torrent.erl b/src/sphinx_builder/sphinx_torrent.erl index 3e105c0..424478c 100644 --- a/src/sphinx_builder/sphinx_torrent.erl +++ b/src/sphinx_builder/sphinx_torrent.erl @@ -112,7 +112,7 @@ forward_date(Date) -> % will cause lots of queries do_load_torrents(Date, Size) -> - Q = {created_at, {'$gt', Date, '$lt', Date + ?DATE_RANGE}}, + Q = {created_at, {'$gt', Date, '$lte', Date + ?DATE_RANGE}}, Conn = mongo_pool:get(?POOLNAME), mongo:do(safe, master, Conn, ?DBNAME, fun() -> % 1: cost lots of memory even close the cursor @@ -122,7 +122,7 @@ do_load_torrents(Date, Size) -> %Cursor = mongo:find(?COLLNAME, {}, {}, Skip), %mongo_cursor:take(Cursor, Size), % 3: - Cursor = mongo:find(?COLLNAME, Q, {}), + Cursor = mongo:find(?COLLNAME, Q, {}, 0, Size), Ret = mongo_cursor:take(Cursor, Size), mongo_cursor:close(Cursor), Ret