diff --git a/src/db_store_mongo.erl b/src/db_store_mongo.erl index fc0a4fd..389d585 100644 --- a/src/db_store_mongo.erl +++ b/src/db_store_mongo.erl @@ -33,7 +33,7 @@ close(Conn) -> mongo_connection:stop(Conn). count(Conn) -> - mongo_do(Conn, fun() -> + mongo_do_slave(Conn, fun() -> mongo:count(?COLLNAME, {}) end). @@ -47,14 +47,14 @@ exist(Conn, Hash) when is_list(Hash) -> search(Conn, Key) when is_list(Key) -> BinColl = list_to_binary(atom_to_list(?COLLNAME)), BinKey = list_to_binary(Key), - Ret = mongo_do(Conn, fun() -> + Ret = mongo_do_slave(Conn, fun() -> mongo:command({text, BinColl, search, BinKey}) end), {decode_search(Ret), decode_search_stats(Ret)}. search_announce_top(Conn, Count) -> Sel = {'$query', {}, '$orderby', {announce, -1}}, - List = mongo_do(Conn, fun() -> + List = mongo_do_slave(Conn, fun() -> % mongodb-erlang does not provide cursor.limit()/sort() functions, wired % but it work here Cursor = mongo:find(?COLLNAME, Sel, [], 0, Count), @@ -65,7 +65,7 @@ search_announce_top(Conn, Count) -> % db.hashes.find({$query:{},$orderby:{created_at: 1}}).limit(10); search_recently(Conn, Count) -> Sel = {'$query', {}, '$orderby', {created_at, -1}}, - List = mongo_do(Conn, fun() -> + List = mongo_do_slave(Conn, fun() -> Cursor = mongo:find(?COLLNAME, Sel, [], 0, Count), mongo_cursor:rest(Cursor) end), @@ -73,14 +73,14 @@ search_recently(Conn, Count) -> search_newest_top(Conn, Count, DaySecs) -> Sel = {'$query', {created_at, {'$gt', DaySecs}}, '$orderby', {announce, -1}}, - List = mongo_do(Conn, fun() -> + List = mongo_do_slave(Conn, fun() -> Cursor = mongo:find(?COLLNAME, Sel, [], 0, Count), mongo_cursor:rest(Cursor) end), [decode_torrent_item(Item) || Item <- List]. index(Conn, Hash) when is_list(Hash) -> - Ret = mongo_do(Conn, fun() -> + Ret = mongo_do_slave(Conn, fun() -> mongo:find_one(?COLLNAME, {'_id', list_to_binary(Hash)}) end), case Ret of @@ -168,6 +168,9 @@ find_exist(Conn, Hash) -> mongo_do(Conn, Fun) -> mongo:do(safe, master, Conn, ?DBNAME, Fun). +mongo_do_slave(Conn, Fun) -> + mongo:do(safe, slave_ok, Conn, ?DBNAME, Fun). + % TODO: replace this with {'_id', ID} hash_selector(Hash) -> Expr = lists:flatten(io_lib:format("this._id == '~s'", [Hash])), diff --git a/src/db_system.erl b/src/db_system.erl index e2067c3..c728258 100644 --- a/src/db_system.erl +++ b/src/db_system.erl @@ -11,6 +11,7 @@ stats_updated/1, stats_query_inserted/2, stats_day_at/2, + stats_day_at_slave/2, stats_get_peers/1]). -define(DBNAME, dht_system). -define(COLLNAME, system). @@ -87,6 +88,11 @@ stats_day_at(Conn, DaySec) -> stats_ensure_today(DaySec) end). +stats_day_at_slave(Conn, DaySec) -> + mongo:do(safe, slave_ok, Conn, ?DBNAME, fun() -> + stats_ensure_today(DaySec) + end). + stats_ensure_today(TodaySecs) -> case mongo:find_one(?STATS_COLLNAME, {'_id', TodaySecs}) of {} -> diff --git a/src/http_front/crawler_http.erl b/src/http_front/crawler_http.erl index f9b1151..6b1f780 100644 --- a/src/http_front/crawler_http.erl +++ b/src/http_front/crawler_http.erl @@ -13,10 +13,18 @@ terminate/2]). -export([start/0, start/3, + start/1, page_temp/0, stop/0]). -record(state, {html_temp, httpid}). +% start from command line, erl -run crawler_http start localhost 27017 8000 +start([DBHostS, DBPortS, PortS]) -> + DBHost = DBHostS, + DBPort = list_to_integer(DBPortS), + HttpPort = list_to_integer(PortS), + start(DBHost, DBPort, HttpPort). + start(DBHost, DBPort, Port) -> code:add_path("deps/bson/ebin"), code:add_path("deps/mongodb/ebin"), diff --git a/src/http_front/db_frontend.erl b/src/http_front/db_frontend.erl index 384150f..95f764f 100644 --- a/src/http_front/db_frontend.erl +++ b/src/http_front/db_frontend.erl @@ -49,8 +49,8 @@ stats() -> Conn = mongo_pool:get(?DB_POOLNAME), DaySecs = time_util:now_day_seconds(), TorSum = db_store_mongo:count(Conn), - D1 = db_system:stats_day_at(Conn, DaySecs), - D2 = db_system:stats_day_at(Conn, DaySecs - ?ONEDAY_SECS), + D1 = db_system:stats_day_at_slave(Conn, DaySecs), + D2 = db_system:stats_day_at_slave(Conn, DaySecs - ?ONEDAY_SECS), {TorSum, [decode_stats(D1), decode_stats(D2)]}. decode_stats(Stats) -> diff --git a/tools/win_start_http.bat b/tools/win_start_http.bat index caa6e2f..0e76cf2 100644 --- a/tools/win_start_http.bat +++ b/tools/win_start_http.bat @@ -1,2 +1,2 @@ -erl -pa ebin -noshell -s crawler_http start +erl -pa ebin -noshell -run crawler_http start localhost 27017 8000