From cb914fe609d6723ce50cecceb5f9117a5fbea91f Mon Sep 17 00:00:00 2001 From: Kevin Lynx Date: Tue, 23 Jul 2013 21:40:17 +0800 Subject: [PATCH] system stats adjust, add more stats to http front-end --- src/db_system.erl | 39 +++++++++++-------------------- src/hash_reader/hash_download.erl | 3 ++- src/http_front/api.erl | 8 ++++--- src/http_front/db_frontend.erl | 10 +------- src/http_front/http_common.erl | 11 +++++++++ src/http_front/http_handler.erl | 8 ++++--- 6 files changed, 38 insertions(+), 41 deletions(-) diff --git a/src/db_system.erl b/src/db_system.erl index 9b430a4..3e2483b 100644 --- a/src/db_system.erl +++ b/src/db_system.erl @@ -10,8 +10,8 @@ -export([stats_new_saved/1, stats_updated/1, stats_query_inserted/2, - stats_day_at/2, stats_day_at_slave/2, + stats_filtered/1, stats_get_peers/1]). -export([get_torrent_id/1]). -compile(export_all). @@ -93,38 +93,27 @@ stats_query_inserted(Conn, Count) -> stats_cache_query_inserted(Conn, Count) -> stats_inc_field(Conn, inserted_query, Count). +stats_filtered(Conn) -> + stats_inc_field(Conn, filter_hash). + stats_inc_field(Conn, Filed) -> stats_inc_field(Conn, Filed, 1). -stats_inc_field(Conn, Filed, Inc) -> +stats_inc_field(Conn, Field, Inc) -> TodaySecs = time_util:now_day_seconds(), - mongo:do(unsafe, master, Conn, ?DBNAME, fun() -> - Doc = stats_ensure_today(TodaySecs), - {Val} = bson:lookup(Filed, Doc), - NewDoc = bson:update(Filed, Val + Inc, Doc), - mongo:update(?STATS_COLLNAME, {'_id', TodaySecs}, NewDoc) - end). - -stats_day_at(Conn, DaySec) -> + Cmd = {findAndModify, ?STATS_COLLNAME, query, {'_id', TodaySecs}, + upsert, true, update, {'$inc', {Field, Inc}}, field, {'_id', 1}}, mongo:do(safe, master, Conn, ?DBNAME, fun() -> - stats_ensure_today(DaySec) + mongo:command(Cmd) end). stats_day_at_slave(Conn, DaySec) -> - mongo:do(safe, slave_ok, Conn, ?DBNAME, fun() -> - stats_ensure_today(DaySec) - end). - -stats_ensure_today(TodaySecs) -> - case mongo:find_one(?STATS_COLLNAME, {'_id', TodaySecs}) of - {} -> - NewDoc = {'_id', TodaySecs, get_peers, 0, get_peers_query, 0, - inserted_query, 0, % because has_cache_writer will merge some queries - updated, 0, new_saved, 0}, - mongo:insert(?STATS_COLLNAME, NewDoc), - NewDoc; - {Doc} -> - Doc + Ret = mongo:do(safe, slave_ok, Conn, ?DBNAME, fun() -> + mongo:find_one(?STATS_COLLNAME, {'_id', DaySec}) + end), + case Ret of + {} -> {}; + {Doc} -> Doc end. %% diff --git a/src/hash_reader/hash_download.erl b/src/hash_reader/hash_download.erl index 3808444..ce635e7 100644 --- a/src/hash_reader/hash_download.erl +++ b/src/hash_reader/hash_download.erl @@ -104,8 +104,9 @@ schedule_download(Conn, Pid, Hash) -> end, try_download(Down, Conn, Pid, Hash). -try_download(false, _, _, Hash) -> +try_download(false, Conn, _, Hash) -> ?T(?FMT("hash does not exist in index_cache, filter it ~s", [Hash])), + db_system:stats_filtered(Conn), hash_reader_stats:handle_cache_filtered(), 0; try_download(true, Conn, Pid, Hash) -> diff --git a/src/http_front/api.erl b/src/http_front/api.erl index 697bb08..7f66e28 100644 --- a/src/http_front/api.erl +++ b/src/http_front/api.erl @@ -126,9 +126,11 @@ do_format_stats([First|Rest]) -> S = [format_stats(First)] ++ ["," ++ format_stats(Stats) || Stats <- Rest], lists:flatten(S). -format_stats({DaySec, Processed, RecvQuery, Updated, New}) -> - ?TEXT("{\"day_secs\":~p, \"recv\":~p, \"process\":~p, \"update\":~p, \"new\":~p}", - [DaySec, RecvQuery, Processed, Updated, New]). +format_stats(Stats) -> + Vals = http_common:stats_to_list(Stats), + ?TEXT("{\"day_secs\":~p, \"recv\":~p, \"process\":~p, \"update\":~p, \"new\":~p, + \"unique\":~p, \"filtered\":~p}", + Vals). %% test_search(Keyword) -> diff --git a/src/http_front/db_frontend.erl b/src/http_front/db_frontend.erl index 48db28f..8d047f5 100644 --- a/src/http_front/db_frontend.erl +++ b/src/http_front/db_frontend.erl @@ -54,15 +54,7 @@ stats() -> D1 = db_system:stats_day_at_slave(Conn, DaySecs), D2 = db_system:stats_day_at_slave(Conn, DaySecs - ?ONEDAY_SECS), D3 = db_system:stats_day_at_slave(Conn, DaySecs - 2 * ?ONEDAY_SECS), - {TorSum, [decode_stats(D1), decode_stats(D2), decode_stats(D3)]}. - -decode_stats(Stats) -> - {DaySec} = bson:lookup('_id', Stats), - {Processed} = bson:lookup(get_peers, Stats), - {RecvQuery} = bson:lookup(get_peers_query, Stats), - {Updated} = bson:lookup(updated, Stats), - {New} = bson:lookup(new_saved, Stats), - {DaySec, Processed, RecvQuery, Updated, New}. + {TorSum, [D1, D2, D3]}. % test only all_top() -> diff --git a/src/http_front/http_common.erl b/src/http_front/http_common.erl index 74cd965..e6f6569 100644 --- a/src/http_front/http_common.erl +++ b/src/http_front/http_common.erl @@ -8,6 +8,8 @@ get_view_hash/1, remote_addr/1, list_to_utf_binary/1, + lookup_stats_item/2, + stats_to_list/1, sort_file_by_size/1]). remote_addr(Env) -> @@ -42,4 +44,13 @@ list_to_utf_binary(L) -> US = unicode:characters_to_binary(UL), US. +lookup_stats_item(Stats, Field) -> + case bson:lookup(Field, Stats) of + {} -> 0; + {Val} -> Val + end. +stats_to_list(Stats) -> + Fileds = ['_id', get_peers_query, get_peers, updated, new_saved, + inserted_query, filter_hash], + [lookup_stats_item(Stats, F) || F <- Fileds]. diff --git a/src/http_front/http_handler.erl b/src/http_front/http_handler.erl index d7db7ff..0479ae8 100644 --- a/src/http_front/http_handler.erl +++ b/src/http_front/http_handler.erl @@ -152,9 +152,11 @@ format_magnet(MagHash) -> format_stats([]) -> []; -format_stats([{DaySec, Processed, RecvQuery, Updated, New}|Rest]) -> - ?TEXT("
  • ~s RecvQuery ~p ProcessedQuery ~p Updated ~p New ~p
    • ", - [format_date_string(DaySec), RecvQuery, Processed, Updated, New]) ++ +format_stats([Stats|Rest]) -> + [DaySec|Vals] = http_common:stats_to_list(Stats), + ?TEXT("
  • ~s RecvQuery ~p ProcessedQuery ~p Updated ~p New ~p + UniqueQuery ~p CacheFiltered ~p
    • ", + [format_date_string(DaySec)|Vals]) ++ format_stats(Rest). format_time_string(Secs) ->