change http to read data from mongodb slave db in repla set, can still

read from master

src/db_store_mongo.erl

@@ -33,7 +33,7 @@ close(Conn) ->
 	mongo_connection:stop(Conn).
 
 count(Conn) ->
-	mongo_do(Conn, fun() ->
+	mongo_do_slave(Conn, fun() ->
 		mongo:count(?COLLNAME, {})
 	end).
 
@@ -47,14 +47,14 @@ exist(Conn, Hash) when is_list(Hash) ->
 search(Conn, Key) when is_list(Key) ->
 	BinColl = list_to_binary(atom_to_list(?COLLNAME)),
 	BinKey = list_to_binary(Key),
-	Ret = mongo_do(Conn, fun() ->
+	Ret = mongo_do_slave(Conn, fun() ->
 		mongo:command({text, BinColl, search, BinKey})
 	end),
 	{decode_search(Ret), decode_search_stats(Ret)}.
 
 search_announce_top(Conn, Count) ->
 	Sel = {'$query', {}, '$orderby', {announce, -1}},
-	List = mongo_do(Conn, fun() ->
+	List = mongo_do_slave(Conn, fun() ->
 		% mongodb-erlang does not provide cursor.limit()/sort() functions, wired
 		% but it work here
 		Cursor = mongo:find(?COLLNAME, Sel, [], 0, Count), 
@@ -65,7 +65,7 @@ search_announce_top(Conn, Count) ->
 % db.hashes.find({$query:{},$orderby:{created_at: 1}}).limit(10);
 search_recently(Conn, Count) ->
 	Sel = {'$query', {}, '$orderby', {created_at, -1}},
-	List = mongo_do(Conn, fun() ->
+	List = mongo_do_slave(Conn, fun() ->
 		Cursor = mongo:find(?COLLNAME, Sel, [], 0, Count), 
 		mongo_cursor:rest(Cursor)
 	end),
@@ -73,14 +73,14 @@ search_recently(Conn, Count) ->
 
 search_newest_top(Conn, Count, DaySecs) ->	
 	Sel = {'$query', {created_at, {'$gt', DaySecs}}, '$orderby', {announce, -1}},
-	List = mongo_do(Conn, fun() ->
+	List = mongo_do_slave(Conn, fun() ->
 		Cursor = mongo:find(?COLLNAME, Sel, [], 0, Count), 
 		mongo_cursor:rest(Cursor)
 	end),
 	[decode_torrent_item(Item) || Item <- List].
 
 index(Conn, Hash) when is_list(Hash) ->
-	Ret = mongo_do(Conn, fun() ->
+	Ret = mongo_do_slave(Conn, fun() ->
 		mongo:find_one(?COLLNAME, {'_id', list_to_binary(Hash)})
 	end),
 	case Ret of 
@@ -168,6 +168,9 @@ find_exist(Conn, Hash) ->
 mongo_do(Conn, Fun) ->
 	mongo:do(safe, master, Conn, ?DBNAME, Fun).
 
+mongo_do_slave(Conn, Fun) ->
+	mongo:do(safe, slave_ok, Conn, ?DBNAME, Fun).
+
 % TODO: replace this with {'_id', ID}
 hash_selector(Hash) ->
 	Expr = lists:flatten(io_lib:format("this._id == '~s'", [Hash])),

src/db_system.erl

@@ -11,6 +11,7 @@
 		 stats_updated/1,
 		 stats_query_inserted/2,
 		 stats_day_at/2,
+		 stats_day_at_slave/2,
 		 stats_get_peers/1]).
 -define(DBNAME, dht_system).
 -define(COLLNAME, system).
@@ -87,6 +88,11 @@ stats_day_at(Conn, DaySec) ->
 		stats_ensure_today(DaySec)
 	end).
 
+stats_day_at_slave(Conn, DaySec) ->
+	mongo:do(safe, slave_ok, Conn, ?DBNAME, fun() ->
+		stats_ensure_today(DaySec)
+	end).
+
 stats_ensure_today(TodaySecs) ->
 	case mongo:find_one(?STATS_COLLNAME, {'_id', TodaySecs}) of
 		{} ->

src/http_front/crawler_http.erl

@@ -13,10 +13,18 @@
 		 terminate/2]).
 -export([start/0, 
 		 start/3,
+		 start/1,
 		 page_temp/0,
 	     stop/0]).
 -record(state, {html_temp, httpid}).
 
+% start from command line, erl -run crawler_http start localhost 27017 8000
+start([DBHostS, DBPortS, PortS]) ->
+	DBHost = DBHostS,
+	DBPort = list_to_integer(DBPortS),
+	HttpPort = list_to_integer(PortS),
+	start(DBHost, DBPort, HttpPort).
+
 start(DBHost, DBPort, Port) ->
 	code:add_path("deps/bson/ebin"),
 	code:add_path("deps/mongodb/ebin"),

src/http_front/db_frontend.erl

@@ -49,8 +49,8 @@ stats() ->
 	Conn = mongo_pool:get(?DB_POOLNAME),
 	DaySecs = time_util:now_day_seconds(),
 	TorSum = db_store_mongo:count(Conn),
-	D1 = db_system:stats_day_at(Conn, DaySecs),
+	D1 = db_system:stats_day_at_slave(Conn, DaySecs),
-	D2 = db_system:stats_day_at(Conn, DaySecs - ?ONEDAY_SECS),
+	D2 = db_system:stats_day_at_slave(Conn, DaySecs - ?ONEDAY_SECS),
 	{TorSum, [decode_stats(D1), decode_stats(D2)]}.
 
 decode_stats(Stats) ->

tools/win_start_http.bat

@@ -1,2 +1,2 @@
-erl -pa ebin -noshell -s crawler_http start
+erl -pa ebin -noshell -run crawler_http start localhost 27017 8000