From a0e1be291d9d55a969355e77d9d6647964b4219c Mon Sep 17 00:00:00 2001
From: Kevin Lynx <kevinlynx@gmail.com>
Date: Mon, 15 Jul 2013 13:45:25 +0800
Subject: [PATCH] add progress displaying for cache_indexer

---
 src/cache_indexer/index_file.erl | 20 +++++++++++++++----
 tools/download_sync.bat          | 34 ++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 4 deletions(-)
 create mode 100644 tools/download_sync.bat

diff --git a/src/cache_indexer/index_file.erl b/src/cache_indexer/index_file.erl
index 5338f3e..1503124 100644
--- a/src/cache_indexer/index_file.erl
+++ b/src/cache_indexer/index_file.erl
@@ -6,6 +6,7 @@
 -module(index_file).
 -export([start/2]).
 -export([worker_run/3]).
+-define(PRINT_STEP, 5000*41).
 
 start(Conn, FileName) ->
 	spawn_link(?MODULE, worker_run, [self(), Conn, FileName]).
@@ -13,6 +14,7 @@ start(Conn, FileName) ->
 load_position(Name) ->
 	StatusFile = Name ++ ".sta",
 	Pos = case file:consult(StatusFile) of
+		{ok, []} -> 0;
 		{ok, [Status]} ->
 			proplists:get_value(position, Status);
 		{error, _} ->
@@ -29,22 +31,32 @@ worker_run(Parent, Conn, FileName) ->
 	Pos = load_position(FileName),
 	io:format("start to process ~s from ~p~n", [FileName, Pos]),
 	{ok, FP} = file:open(FileName, [read]),
+	{ok, MaxPos} = file:position(FP, eof),
 	file:position(FP, Pos),
-	Sum = process_hash(Conn, FileName, FP),
+	Step = MaxPos div 10, % every 10% to display progress
+	CheckStep = if Step < ?PRINT_STEP -> ?PRINT_STEP; true -> Step end,
+	Sum = process_hash(Conn, FileName, FP, Pos, MaxPos, CheckStep),
 	Parent ! {worker_done, self(), FileName},
 	file:close(FP),
 	io:format("Index file ~s done, ~p hashes~n", [FileName, Sum]).
 
-process_hash(Conn, FileName, FP) ->
+process_hash(Conn, FileName, FP, PrintPos, MaxPos, CheckStep) ->
 	case io:get_line(FP, "") of
 		eof -> 0;
 		Line ->
 			save_hash(Conn, strip_lf(Line)),
 			{ok, Pos} = file:position(FP, cur),
-			save_position(FileName, Pos),
-			1 + process_hash(Conn, FileName, FP)
+			NewPrintPos = check_progress(FileName, Pos, PrintPos, MaxPos, CheckStep),
+			1 + process_hash(Conn, FileName, FP, NewPrintPos, MaxPos, CheckStep)
 	end.
 
+check_progress(FileName, Pos, PrintPos, MaxPos, CheckStep) when Pos >= PrintPos ->
+	save_position(FileName, Pos),
+	io:format("~s -> ~b%~n", [FileName, 100 * Pos div MaxPos]),
+	PrintPos + CheckStep;
+check_progress(_, _, PrintPos, _, _) ->
+	PrintPos.
+
 strip_lf(S) ->
 	lists:sublist(S, length(S) - 1).
 
diff --git a/tools/download_sync.bat b/tools/download_sync.bat
new file mode 100644
index 0000000..48fc9bd
--- /dev/null
+++ b/tools/download_sync.bat
@@ -0,0 +1,34 @@
+wget http://torrage.com/sync/201201.txt
+wget http://torrage.com/sync/201202.txt
+wget http://torrage.com/sync/201203.txt
+wget http://torrage.com/sync/201204.txt
+wget http://torrage.com/sync/201205.txt
+wget http://torrage.com/sync/201206.txt
+wget http://torrage.com/sync/201207.txt
+wget http://torrage.com/sync/201208.txt
+wget http://torrage.com/sync/201209.txt
+wget http://torrage.com/sync/201210.txt
+wget http://torrage.com/sync/201211.txt
+wget http://torrage.com/sync/201212.txt
+wget http://torrage.com/sync/201301.txt
+wget http://torrage.com/sync/201302.txt
+wget http://torrage.com/sync/201303.txt
+wget http://torrage.com/sync/201304.txt
+wget http://torrage.com/sync/201305.txt
+wget http://torrage.com/sync/201306.txt
+wget http://torrage.com/sync/20130701.txt
+wget http://torrage.com/sync/20130702.txt
+wget http://torrage.com/sync/20130703.txt
+wget http://torrage.com/sync/20130704.txt
+wget http://torrage.com/sync/20130705.txt
+wget http://torrage.com/sync/20130706.txt
+wget http://torrage.com/sync/20130707.txt
+wget http://torrage.com/sync/20130708.txt
+wget http://torrage.com/sync/20130709.txt
+wget http://torrage.com/sync/20130710.txt
+wget http://torrage.com/sync/20130711.txt
+wget http://torrage.com/sync/20130712.txt
+wget http://torrage.com/sync/20130713.txt
+wget http://torrage.com/sync/20130714.txt
+wget http://torrage.com/sync/20130715.txt
+