Mirror/dhtcrawler2
1
0
Fork 0
mirror of https://github.com/btdig/dhtcrawler2.git synced 2025-02-23 13:49:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

change http startup

Browse Source
This commit is contained in:
Kevin Lynx 2013-07-09 22:42:40 +08:00
parent 46338b02c6
commit ae9f2f0c8e
3 changed files with 15 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/http_front/crawler_http.erl
View File

@ -12,28 +12,29 @@
code_change/3,
terminate/2]).
-export([start/0,
start/3,
start/4,
start/1,
page_temp/0,
stop/0]).
-record(state, {html_temp, httpid}).
% start from command line, erl -run crawler_http start localhost 27017 8000
start([DBHostS, DBPortS, PortS]) ->
% start from command line, erl -run crawler_http start localhost 27017 8000 5
start([DBHostS, DBPortS, PortS, PoolSizeS]) ->
DBHost = DBHostS,
DBPort = list_to_integer(DBPortS),
HttpPort = list_to_integer(PortS),
start(DBHost, DBPort, HttpPort).
PoolSize = list_to_integer(PoolSizeS),
start(DBHost, DBPort, HttpPort, PoolSize).
start(DBHost, DBPort, Port) ->
start(DBHost, DBPort, Port, PoolSize) ->
code:add_path("deps/bson/ebin"),
code:add_path("deps/mongodb/ebin"),
Apps = [crypto, public_key, ssl, inets, bson, mongodb],
[application:start(App) || App <- Apps],
gen_server:start({local, srv_name()}, ?MODULE, [DBHost, DBPort, Port], []).
gen_server:start({local, srv_name()}, ?MODULE, [DBHost, DBPort, Port, PoolSize], []).
start() ->
start(localhost, 27017, 8000).
start(localhost, 27017, 8000, 5).
stop() ->
gen_server:cast(srv_name(), stop).
@ -44,9 +45,9 @@ page_temp() ->
srv_name() ->
crawler_http.
init([DBHost, DBPort, Port]) ->
init([DBHost, DBPort, Port, PoolSize]) ->
process_flag(trap_exit, true),
db_frontend:start(DBHost, DBPort, 2),
db_frontend:start(DBHost, DBPort, PoolSize),
http_cache:start_link(),
{ok, Pid} = inets:start(httpd, [
{modules, [mod_alias, mod_auth, mod_esi, mod_actions,

5
src/http_front/http_cache.erl
View File

@ -93,9 +93,12 @@ query(Type, State) ->
end.
update(Type, #state{cache = Cache} = State) ->
Start = now(),
io:format("sync update cache ~p start~n", [Type]),
Ret = do_update(Type),
Val = {now(), Ret},
io:format("sync update cache ~p~n", [Type]),
io:format("sync update cache ~p done used ~p ms~n", [Type,
timer:now_diff(now(), Start) div 1000]),
NewCache = gb_trees:enter(Type, Val, Cache),
case gb_trees:size(NewCache) >= ?CACHE_SIZE of
true ->

2
tools/win_start_http.bat
View File

@ -1,2 +1,2 @@
erl -pa ebin -noshell -run crawler_http start localhost 27017 8000
erl -pa ebin -noshell -run crawler_http start localhost 27017 8000 5