diff --git a/README.md b/README.md index d18f843..a68a7a9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,14 @@ ## dhtcrawler2 -This git branch maintain pre-compiled erlang files to start dhtcrawler2 directly. +dhtcrawler is a DHT crawler written in erlang. It can join a DHT network and crawl many P2P torrents. The program save all torrent info into database and provide an http interface to search a torrent by a keyword. + +![screenshot](https://raw.github.com/kevinlynx/dhtcrawler/master/screenshot.png) + +dhtcrawler2 is an extended version to [dhtcrawler](https://github.com/kevinlynx/dhtcrawler). It has improved a lot on crawling speed, and much more stable. + +This git branch maintain pre-compiled erlang files to start dhtcrawler2 directly. So you don't need to compile it yourself, just download it and run it to collect torrents and search a torrent by a keyword. + +Enjoy it! ## Usage @@ -14,5 +22,19 @@ This git branch maintain pre-compiled erlang files to start dhtcrawler2 directly * start **httpd**, on Windows, just click `win_start_http.bat` * wait several minutes and checkout `localhost:8000` +You can also compile the source code and run it manually. The source code is in `src` branch of this repo. +Also you can check more technique information at my blog site (Chinese) [codemacro.com](http://codemacro.com) + +## Config + +Most config value is in `priv/dhtcrawler.config`, when you first run dhtcrawler, this file will be generated automatically. And the other config values are passed by arguments to erlang functions. In most case you don't need to change these config values, except these network addresses. + +## Mongodb Repla set + +It's not related about dhtcrawler, but only Mongodb, try figure it yourself. + +## Another http front-end + +Yes of course you can write another http front-end UI based on the torrent database, if you're interested in it I can help you about the database format. diff --git a/deps/bson/ebin/bson.beam b/deps/bson/ebin/bson.beam index adc350d..af6f0bf 100644 Binary files a/deps/bson/ebin/bson.beam and b/deps/bson/ebin/bson.beam differ diff --git a/deps/bson/ebin/bson_binary.beam b/deps/bson/ebin/bson_binary.beam index d9363a2..c21b23f 100644 Binary files a/deps/bson/ebin/bson_binary.beam and b/deps/bson/ebin/bson_binary.beam differ diff --git a/deps/bson/ebin/bson_tests.beam b/deps/bson/ebin/bson_tests.beam index c7d291a..9f8a5f5 100644 Binary files a/deps/bson/ebin/bson_tests.beam and b/deps/bson/ebin/bson_tests.beam differ diff --git a/deps/ibrowse/ebin/ibrowse.beam b/deps/ibrowse/ebin/ibrowse.beam index b5dae70..54e2aa6 100644 Binary files a/deps/ibrowse/ebin/ibrowse.beam and b/deps/ibrowse/ebin/ibrowse.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_app.beam b/deps/ibrowse/ebin/ibrowse_app.beam index 7175c02..62fb1d5 100644 Binary files a/deps/ibrowse/ebin/ibrowse_app.beam and b/deps/ibrowse/ebin/ibrowse_app.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_http_client.beam b/deps/ibrowse/ebin/ibrowse_http_client.beam index 9c1b22c..34ed0ad 100644 Binary files a/deps/ibrowse/ebin/ibrowse_http_client.beam and b/deps/ibrowse/ebin/ibrowse_http_client.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_lb.beam b/deps/ibrowse/ebin/ibrowse_lb.beam index 8ab7017..e920334 100644 Binary files a/deps/ibrowse/ebin/ibrowse_lb.beam and b/deps/ibrowse/ebin/ibrowse_lb.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_lib.beam b/deps/ibrowse/ebin/ibrowse_lib.beam index ffe5172..1f9d905 100644 Binary files a/deps/ibrowse/ebin/ibrowse_lib.beam and b/deps/ibrowse/ebin/ibrowse_lib.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_sup.beam b/deps/ibrowse/ebin/ibrowse_sup.beam index cc71806..0f74506 100644 Binary files a/deps/ibrowse/ebin/ibrowse_sup.beam and b/deps/ibrowse/ebin/ibrowse_sup.beam differ diff --git a/deps/kdht/ebin/bencode.beam b/deps/kdht/ebin/bencode.beam index 2c371eb..5080e66 100644 Binary files a/deps/kdht/ebin/bencode.beam and b/deps/kdht/ebin/bencode.beam differ diff --git a/deps/kdht/ebin/bucket.beam b/deps/kdht/ebin/bucket.beam index 9a63aaf..5189cfc 100644 Binary files a/deps/kdht/ebin/bucket.beam and b/deps/kdht/ebin/bucket.beam differ diff --git a/deps/kdht/ebin/dht_id.beam b/deps/kdht/ebin/dht_id.beam index 775ab7d..5f7e578 100644 Binary files a/deps/kdht/ebin/dht_id.beam and b/deps/kdht/ebin/dht_id.beam differ diff --git a/deps/kdht/ebin/dht_net.beam b/deps/kdht/ebin/dht_net.beam index 5c58ed5..aff964a 100644 Binary files a/deps/kdht/ebin/dht_net.beam and b/deps/kdht/ebin/dht_net.beam differ diff --git a/deps/kdht/ebin/dht_state.beam b/deps/kdht/ebin/dht_state.beam index 2cd8b01..1b115a2 100644 Binary files a/deps/kdht/ebin/dht_state.beam and b/deps/kdht/ebin/dht_state.beam differ diff --git a/deps/kdht/ebin/kdht_sup.beam b/deps/kdht/ebin/kdht_sup.beam index d38eb00..5f3ec2c 100644 Binary files a/deps/kdht/ebin/kdht_sup.beam and b/deps/kdht/ebin/kdht_sup.beam differ diff --git a/deps/kdht/ebin/msg.beam b/deps/kdht/ebin/msg.beam index 181a136..890f988 100644 Binary files a/deps/kdht/ebin/msg.beam and b/deps/kdht/ebin/msg.beam differ diff --git a/deps/kdht/ebin/search.beam b/deps/kdht/ebin/search.beam index 82a2c68..f630f7f 100644 Binary files a/deps/kdht/ebin/search.beam and b/deps/kdht/ebin/search.beam differ diff --git a/deps/kdht/ebin/storage.beam b/deps/kdht/ebin/storage.beam index 0d2b0bc..cf7229b 100644 Binary files a/deps/kdht/ebin/storage.beam and b/deps/kdht/ebin/storage.beam differ diff --git a/deps/kdht/ebin/timer_monitor.beam b/deps/kdht/ebin/timer_monitor.beam index 0d19e07..2369cde 100644 Binary files a/deps/kdht/ebin/timer_monitor.beam and b/deps/kdht/ebin/timer_monitor.beam differ diff --git a/deps/kdht/ebin/vlog.beam b/deps/kdht/ebin/vlog.beam index 3e7c5d9..14706c9 100644 Binary files a/deps/kdht/ebin/vlog.beam and b/deps/kdht/ebin/vlog.beam differ diff --git a/deps/mongodb/ebin/mongo.beam b/deps/mongodb/ebin/mongo.beam index 536ba6c..b7c4d61 100644 Binary files a/deps/mongodb/ebin/mongo.beam and b/deps/mongodb/ebin/mongo.beam differ diff --git a/deps/mongodb/ebin/mongo_app.beam b/deps/mongodb/ebin/mongo_app.beam index 8a13224..190225e 100644 Binary files a/deps/mongodb/ebin/mongo_app.beam and b/deps/mongodb/ebin/mongo_app.beam differ diff --git a/deps/mongodb/ebin/mongo_connection.beam b/deps/mongodb/ebin/mongo_connection.beam index 10242bd..982951d 100644 Binary files a/deps/mongodb/ebin/mongo_connection.beam and b/deps/mongodb/ebin/mongo_connection.beam differ diff --git a/deps/mongodb/ebin/mongo_cursor.beam b/deps/mongodb/ebin/mongo_cursor.beam index d2baa54..58cf89e 100644 Binary files a/deps/mongodb/ebin/mongo_cursor.beam and b/deps/mongodb/ebin/mongo_cursor.beam differ diff --git a/deps/mongodb/ebin/mongo_id_server.beam b/deps/mongodb/ebin/mongo_id_server.beam index 1e8bb6e..ef86195 100644 Binary files a/deps/mongodb/ebin/mongo_id_server.beam and b/deps/mongodb/ebin/mongo_id_server.beam differ diff --git a/deps/mongodb/ebin/mongo_pool.beam b/deps/mongodb/ebin/mongo_pool.beam index 9e3841e..d4a7d6d 100644 Binary files a/deps/mongodb/ebin/mongo_pool.beam and b/deps/mongodb/ebin/mongo_pool.beam differ diff --git a/deps/mongodb/ebin/mongo_protocol.beam b/deps/mongodb/ebin/mongo_protocol.beam index e96fdb4..3ecf7b7 100644 Binary files a/deps/mongodb/ebin/mongo_protocol.beam and b/deps/mongodb/ebin/mongo_protocol.beam differ diff --git a/deps/mongodb/ebin/mongo_sup.beam b/deps/mongodb/ebin/mongo_sup.beam index be18196..ce8c0e7 100644 Binary files a/deps/mongodb/ebin/mongo_sup.beam and b/deps/mongodb/ebin/mongo_sup.beam differ diff --git a/ebin/bt_conn.beam b/ebin/bt_conn.beam index f07aa66..b6b9010 100644 Binary files a/ebin/bt_conn.beam and b/ebin/bt_conn.beam differ diff --git a/ebin/bt_message.beam b/ebin/bt_message.beam index 74bb87a..8fd1b8c 100644 Binary files a/ebin/bt_message.beam and b/ebin/bt_message.beam differ diff --git a/ebin/crawler_app.beam b/ebin/crawler_app.beam index 13d4067..83f53cc 100644 Binary files a/ebin/crawler_app.beam and b/ebin/crawler_app.beam differ diff --git a/ebin/crawler_http.beam b/ebin/crawler_http.beam index fdd42b4..2210405 100644 Binary files a/ebin/crawler_http.beam and b/ebin/crawler_http.beam differ diff --git a/ebin/crawler_stats.beam b/ebin/crawler_stats.beam index e8ba8be..2c10bb5 100644 Binary files a/ebin/crawler_stats.beam and b/ebin/crawler_stats.beam differ diff --git a/ebin/crawler_sup.beam b/ebin/crawler_sup.beam index a45b35b..4609f59 100644 Binary files a/ebin/crawler_sup.beam and b/ebin/crawler_sup.beam differ diff --git a/ebin/db_frontend.beam b/ebin/db_frontend.beam index f9bb57d..3746a3b 100644 Binary files a/ebin/db_frontend.beam and b/ebin/db_frontend.beam differ diff --git a/ebin/db_hash.beam b/ebin/db_hash.beam index 1c38372..fb2703c 100644 Binary files a/ebin/db_hash.beam and b/ebin/db_hash.beam differ diff --git a/ebin/db_hash_reader.beam b/ebin/db_hash_reader.beam index 1e22a54..4ee5df1 100644 Binary files a/ebin/db_hash_reader.beam and b/ebin/db_hash_reader.beam differ diff --git a/ebin/db_hash_reader_sup.beam b/ebin/db_hash_reader_sup.beam index 385b06f..000ab2f 100644 Binary files a/ebin/db_hash_reader_sup.beam and b/ebin/db_hash_reader_sup.beam differ diff --git a/ebin/db_store_mongo.beam b/ebin/db_store_mongo.beam index 6796085..2cdcb1b 100644 Binary files a/ebin/db_store_mongo.beam and b/ebin/db_store_mongo.beam differ diff --git a/ebin/db_system.beam b/ebin/db_system.beam index 150d3af..6da376f 100644 Binary files a/ebin/db_system.beam and b/ebin/db_system.beam differ diff --git a/ebin/dht_monitor.beam b/ebin/dht_monitor.beam index 6f9547d..75b8afb 100644 Binary files a/ebin/dht_monitor.beam and b/ebin/dht_monitor.beam differ diff --git a/ebin/dhtcrawler.app b/ebin/dhtcrawler.app index 9ec3921..0de64bc 100644 --- a/ebin/dhtcrawler.app +++ b/ebin/dhtcrawler.app @@ -1,6 +1,6 @@ {application,dhtcrawler, [{description,"A DHT crawler to index magnet hash to torrent"}, - {vsn,"3505116"}, + {vsn,"c82f006"}, {registered,[dht_crawler_sup]}, {applications,[kernel,stdlib,crypto,public_key,ssl,inets,bson, mongodb]}, diff --git a/ebin/hash_reader_stats.beam b/ebin/hash_reader_stats.beam index d30da03..62b9417 100644 Binary files a/ebin/hash_reader_stats.beam and b/ebin/hash_reader_stats.beam differ diff --git a/ebin/http_handler.beam b/ebin/http_handler.beam index 8bc2d37..3868f12 100644 Binary files a/ebin/http_handler.beam and b/ebin/http_handler.beam differ diff --git a/ebin/string_split.beam b/ebin/string_split.beam index daf9506..5ed8fe5 100644 Binary files a/ebin/string_split.beam and b/ebin/string_split.beam differ diff --git a/ebin/time_util.beam b/ebin/time_util.beam index efb3139..bced3e7 100644 Binary files a/ebin/time_util.beam and b/ebin/time_util.beam differ diff --git a/ebin/tor_download.beam b/ebin/tor_download.beam index b0c301f..3e633f3 100644 Binary files a/ebin/tor_download.beam and b/ebin/tor_download.beam differ diff --git a/ebin/tor_download_stats.beam b/ebin/tor_download_stats.beam index 7756ce8..02d3675 100644 Binary files a/ebin/tor_download_stats.beam and b/ebin/tor_download_stats.beam differ diff --git a/ebin/torrent_file.beam b/ebin/torrent_file.beam index 75b0dea..9fad45a 100644 Binary files a/ebin/torrent_file.beam and b/ebin/torrent_file.beam differ diff --git a/ebin/urldecode.beam b/ebin/urldecode.beam index 24d3c9e..9d0f13d 100644 Binary files a/ebin/urldecode.beam and b/ebin/urldecode.beam differ diff --git a/ebin/vlog.beam b/ebin/vlog.beam index b7a9db5..d67dbec 100644 Binary files a/ebin/vlog.beam and b/ebin/vlog.beam differ