first commit

2024-12-12 19:46:22 +00:00 · 2013-07-01 22:42:14 +08:00 · 2013-07-01 22:42:14 +08:00 · 4f7fe8805e
commit 4f7fe8805e
51 changed files with 213 additions and 0 deletions
--- a/README.md
+++ b/README.md
@ -0,0 +1,18 @@
+## dhtcrawler2
+
+This git branch maintain pre-compiled erlang files to start dhtcrawler2 directly. 
+
+## Usage
+
+* install Erlang R16B or newer
+* download mongodb and start mongodb first
+
+        mongod --dbpath your-database-path --setParameter textSearchEnabled=true
+
+* start **crawler**, on Windows, just click `win_start_crawler.bat`
+* start **hash_reader**, on Windows, just click `win_start_hash.bat`
+* start **httpd**, on Windows, just click `win_start_http.bat`
+* wait several minutes and checkout `localhost:8000`
+
+
+
--- a/create_bin.bat
+++ b/create_bin.bat
@ -0,0 +1,16 @@
+cd ..
+mkdir bin\deps\bson\ebin
+mkdir bin\deps\mongodb\ebin
+mkdir bin\deps\kdht\ebin
+mkdir bin\deps\ibrowse\ebin
+copy deps\bson\ebin\*.* bin\deps\bson\ebin\
+copy deps\mongodb\ebin\*.* bin\deps\mongodb\ebin\
+copy deps\kdht\ebin\*.* bin\deps\kdht\ebin\
+copy deps\ibrowse\ebin\*.* bin\deps\ibrowse\ebin\
+mkdir bin\www
+copy www\*.* bin\www\
+copy tools\*.* bin\
+mkdir bin\priv
+mkdir bin\ebin         
+copy ebin\*.* bin\ebin\
+pause
--- a/deps/bson/ebin/bson.app
+++ b/deps/bson/ebin/bson.app
@ -0,0 +1,7 @@
+{application, bson,
+ [{description, "BSON are JSON-like objects with a standard binary serialization. See bsonspec.org."},
+  {vsn, "0"},
+  {modules, [bson, bson_binary, bson_tests]},
+  {registered, []},
+  {applications, [kernel, stdlib]}
+ ]}.
--- a/deps/bson/ebin/bson.beam
+++ b/deps/bson/ebin/bson.beam
--- a/deps/bson/ebin/bson_binary.beam
+++ b/deps/bson/ebin/bson_binary.beam
--- a/deps/bson/ebin/bson_tests.beam
+++ b/deps/bson/ebin/bson_tests.beam
--- a/deps/ibrowse/ebin/ibrowse.app
+++ b/deps/ibrowse/ebin/ibrowse.app
@ -0,0 +1,9 @@
+{application,ibrowse,
+             [{description,"Erlang HTTP client application"},
+              {vsn,"4.0.1"},
+              {registered,[ibrowse_sup,ibrowse]},
+              {applications,[kernel,stdlib]},
+              {env,[]},
+              {mod,{ibrowse_app,[]}},
+              {modules,[ibrowse,ibrowse_app,ibrowse_http_client,ibrowse_lb,
+                        ibrowse_lib,ibrowse_sup]}]}.
--- a/deps/ibrowse/ebin/ibrowse.beam
+++ b/deps/ibrowse/ebin/ibrowse.beam
--- a/deps/ibrowse/ebin/ibrowse_app.beam
+++ b/deps/ibrowse/ebin/ibrowse_app.beam
--- a/deps/ibrowse/ebin/ibrowse_http_client.beam
+++ b/deps/ibrowse/ebin/ibrowse_http_client.beam
--- a/deps/ibrowse/ebin/ibrowse_lb.beam
+++ b/deps/ibrowse/ebin/ibrowse_lb.beam
--- a/deps/ibrowse/ebin/ibrowse_lib.beam
+++ b/deps/ibrowse/ebin/ibrowse_lib.beam
--- a/deps/ibrowse/ebin/ibrowse_sup.beam
+++ b/deps/ibrowse/ebin/ibrowse_sup.beam
--- a/deps/kdht/ebin/.gitignore
+++ b/deps/kdht/ebin/.gitignore
@ -0,0 +1 @@
+*.beam
--- a/deps/kdht/ebin/kdht.app
+++ b/deps/kdht/ebin/kdht.app
@ -0,0 +1,10 @@
+{application, kdht, [
+  {description, "DHT library"},
+  {vsn, git},
+  {registered, []},
+  {applications, [kernel, stdlib]},
+  {modules, [bencode, bucket, dht_id, dht_net, dht_state, kdht_sup, msg, 
+  	search, storage, timer_monitor, vlog]},
+  {mod, {}}
+]}.
+
--- a/deps/mongodb/ebin/mongo.beam
+++ b/deps/mongodb/ebin/mongo.beam
--- a/deps/mongodb/ebin/mongo_app.beam
+++ b/deps/mongodb/ebin/mongo_app.beam
--- a/deps/mongodb/ebin/mongo_connection.beam
+++ b/deps/mongodb/ebin/mongo_connection.beam
--- a/deps/mongodb/ebin/mongo_cursor.beam
+++ b/deps/mongodb/ebin/mongo_cursor.beam
--- a/deps/mongodb/ebin/mongo_id_server.beam
+++ b/deps/mongodb/ebin/mongo_id_server.beam
--- a/deps/mongodb/ebin/mongo_pool.beam
+++ b/deps/mongodb/ebin/mongo_pool.beam
--- a/deps/mongodb/ebin/mongo_protocol.beam
+++ b/deps/mongodb/ebin/mongo_protocol.beam
--- a/deps/mongodb/ebin/mongo_sup.beam
+++ b/deps/mongodb/ebin/mongo_sup.beam
--- a/deps/mongodb/ebin/mongodb.app
+++ b/deps/mongodb/ebin/mongodb.app
@ -0,0 +1,9 @@
+{application,mongodb,
+             [{description,"Client interface to MongoDB, also known as the driver. See www.mongodb.org"},
+              {vsn,"v0.3.1-96-g613f157"},
+              {registered,[]},
+              {applications,[kernel,stdlib,bson]},
+              {mod,{mongo_app,[]}},
+              {modules,[mongo,mongo_app,mongo_connection,mongo_cursor,
+                        mongo_id_server,mongo_pool,mongo_protocol,
+                        mongo_sup]}]}.
--- a/ebin/bt_conn.beam
+++ b/ebin/bt_conn.beam
--- a/ebin/bt_message.beam
+++ b/ebin/bt_message.beam
--- a/ebin/crawler_app.beam
+++ b/ebin/crawler_app.beam
--- a/ebin/crawler_http.beam
+++ b/ebin/crawler_http.beam
--- a/ebin/crawler_stats.beam
+++ b/ebin/crawler_stats.beam
--- a/ebin/crawler_sup.beam
+++ b/ebin/crawler_sup.beam
--- a/ebin/db_hash.beam
+++ b/ebin/db_hash.beam
--- a/ebin/db_hash_reader.beam
+++ b/ebin/db_hash_reader.beam
--- a/ebin/db_hash_reader_sup.beam
+++ b/ebin/db_hash_reader_sup.beam
--- a/ebin/db_store_mongo.beam
+++ b/ebin/db_store_mongo.beam
--- a/ebin/db_system.beam
+++ b/ebin/db_system.beam
--- a/ebin/dht_monitor.beam
+++ b/ebin/dht_monitor.beam
--- a/ebin/dhtcrawler.app
+++ b/ebin/dhtcrawler.app
@ -0,0 +1,13 @@
+{application,dhtcrawler,
+             [{description,"A DHT crawler to index magnet hash to torrent"},
+              {vsn,"6dde34b"},
+              {registered,[dht_crawler_sup]},
+              {applications,[kernel,stdlib,crypto,public_key,ssl,inets,bson,
+                             mongodb]},
+              {mod,{crawler_app,[]}},
+              {modules,[bt_conn,bt_message,crawler_app,crawler_http,
+                        crawler_stats,crawler_sup,db_hash,db_hash_reader,
+                        db_hash_reader_sup,db_store_mongo,db_system,
+                        dht_monitor,hash_reader_stats,http_handler,
+                        string_split,time_util,tor_download,
+                        tor_download_stats,torrent_file,urldecode,vlog]}]}.
--- a/ebin/hash_reader_stats.beam
+++ b/ebin/hash_reader_stats.beam
--- a/ebin/http_handler.beam
+++ b/ebin/http_handler.beam
--- a/ebin/string_split.beam
+++ b/ebin/string_split.beam
--- a/ebin/time_util.beam
+++ b/ebin/time_util.beam
--- a/ebin/tor_download.beam
+++ b/ebin/tor_download.beam
--- a/ebin/tor_download_stats.beam
+++ b/ebin/tor_download_stats.beam
--- a/ebin/torrent_file.beam
+++ b/ebin/torrent_file.beam
--- a/ebin/urldecode.beam
+++ b/ebin/urldecode.beam
--- a/ebin/vlog.beam
+++ b/ebin/vlog.beam
--- a/win_start_crawler.bat
+++ b/win_start_crawler.bat
@ -0,0 +1,2 @@
+erl -pa ebin -noshell -s crawler_app start
+
--- a/win_start_hash.bat
+++ b/win_start_hash.bat
@ -0,0 +1 @@
+erl -pa ebin -noshell -s db_hash_reader_sup start_standalone localhost 27017 10
--- a/win_start_http.bat
+++ b/win_start_http.bat
@ -0,0 +1,2 @@
+erl -pa ebin -noshell -s crawler_http start
+
--- a/www/index.html
+++ b/www/index.html
@ -0,0 +1,66 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>Yet Another Magnet Search Engine</title>
+<style type="text/css" media="screen">
+body { margin:0; padding:0; }
+#wrap {
+  width: 980px;
+  margin: 5px auto;
+}
+#header {
+  height: 30px;
+}
+#main {
+  margin: 20px 10px;
+}
+#content {
+  min-height: 130px;
+  padding: 20px 5px;
+}
+#footer{
+    text-align:center;
+    font-size: 13px;
+    margin-top: 20px;
+    padding-top: 5px;
+    border-top: 1px solid #888;
+}
+.search-box #s_text {
+   width: 200px;
+}
+</style>
+</head>
+<body>
+<div id="wrap">
+<div id="header">
+ <h2><a style="text-decoration:none;" href="/">Magnet Search Engine</a></h2>
+</div>
+<div id="main">
+<div class="search-box">
+   <form id="s_box" method="get" action="/e/http_handler:search" >
+        <input id="s_text" name="q" type="text" />
+        <input id="s_submit" value="Search" type="submit" />
+    </form>
+</div>
+<div id="content">
+<p>
+Try <a href="/e/http_handler:search?q=avi">AVI</a>  
+&nbsp;<a href="/e/http_handler:search?q=american">American</a>  
+&nbsp;<a href="/e/http_handler:search?q=iron+man">Iron Man</a>  
+</p>
+</div>
+    <div id="footer">
+        <a href="/e/http_handler:recent">New</a>
+        <a href="/e/http_handler:top">Top</a>
+        <a href="/e/http_handler:stats">Stats</a>
+        <span> &copy; Kevin Lynx 2013</span>
+    </div>
+    <script type="text/javascript">
+        document.getElementsByTagName('input')[0].focus();
+    </script>
+</div>
+</div>
+</body>
+</html>
+
--- a/www/page.temp
+++ b/www/page.temp
@ -0,0 +1,59 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>Yet Another Magnet Search Engine</title>
+<style type="text/css" media="screen">
+body { margin:0; padding:0; }
+#wrap {
+  width: 980px;
+  margin: 5px auto;
+}
+#header {
+  height: 30px;
+}
+#main {
+  margin: 20px 10px;
+}
+#content {
+  min-height: 130px;
+  padding: 20px 5px;
+}
+#footer{
+    text-align:center;
+    font-size: 13px;
+    margin-top: 20px;
+    padding-top: 5px;
+    border-top: 1px solid #888;
+}
+.search-box #s_text {
+   width: 200px;
+}
+</style>
+</head>
+<body>
+<div id="wrap">
+<div id="header">
+ <h2><a style="text-decoration:none;" href="/">Magnet Search Engine</a></h2>
+</div>
+<div id="main">
+<div class="search-box">
+   <form id="s_box" method="get" action="/e/http_handler:search" >
+        <input id="s_text" value="~s" name="q" type="text" />
+        <input id="s_submit" value="Search" type="submit" />
+    </form>
+</div>
+<div id="content">
+~s
+</div>
+    <div id="footer">
+        <a href="/e/http_handler:recent">New</a>
+        <a href="/e/http_handler:top">Top</a>
+        <a href="/e/http_handler:stats">Stats</a>
+        <span> &copy; Kevin Lynx 2013</span>
+    </div>
+</div>
+</div>
+</body>
+</html>
+
				`@ -0,0 +1,2 @@`
				`erl -pa ebin -noshell -s crawler_app start`
				`@ -0,0 +1 @@`
				`erl -pa ebin -noshell -s db_hash_reader_sup start_standalone localhost 27017 10`
				`@ -0,0 +1,2 @@`
				`erl -pa ebin -noshell -s crawler_http start`