diff --git a/README.md b/README.md index 7eb4b98..3808f9d 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,10 @@ You can also compile the source code and run it manually. The source code is in Also you can check more technique information at my blog site (Chinese) [codemacro.com](http://codemacro.com) +## Source code + +dhtcrawler is totally open source, and can be used in any purpose, but you should keep my name on, copyright by me please. You can checkout dhtcrawler2 source code in this git repo **src** branch. + ## Config Most config value is in `priv/dhtcrawler.config`, when you first run dhtcrawler, this file will be generated automatically. And the other config values are passed by arguments to erlang functions. In most case you don't need to change these config values, except these network addresses. diff --git a/deps/bson/ebin/bson.beam b/deps/bson/ebin/bson.beam index adc350d..af6f0bf 100644 Binary files a/deps/bson/ebin/bson.beam and b/deps/bson/ebin/bson.beam differ diff --git a/deps/bson/ebin/bson_binary.beam b/deps/bson/ebin/bson_binary.beam index d9363a2..c21b23f 100644 Binary files a/deps/bson/ebin/bson_binary.beam and b/deps/bson/ebin/bson_binary.beam differ diff --git a/deps/bson/ebin/bson_tests.beam b/deps/bson/ebin/bson_tests.beam index c7d291a..9f8a5f5 100644 Binary files a/deps/bson/ebin/bson_tests.beam and b/deps/bson/ebin/bson_tests.beam differ diff --git a/deps/ibrowse/ebin/ibrowse.beam b/deps/ibrowse/ebin/ibrowse.beam index b5dae70..54e2aa6 100644 Binary files a/deps/ibrowse/ebin/ibrowse.beam and b/deps/ibrowse/ebin/ibrowse.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_app.beam b/deps/ibrowse/ebin/ibrowse_app.beam index 7175c02..62fb1d5 100644 Binary files a/deps/ibrowse/ebin/ibrowse_app.beam and b/deps/ibrowse/ebin/ibrowse_app.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_http_client.beam b/deps/ibrowse/ebin/ibrowse_http_client.beam index 9c1b22c..34ed0ad 100644 Binary files a/deps/ibrowse/ebin/ibrowse_http_client.beam and b/deps/ibrowse/ebin/ibrowse_http_client.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_lb.beam b/deps/ibrowse/ebin/ibrowse_lb.beam index 8ab7017..e920334 100644 Binary files a/deps/ibrowse/ebin/ibrowse_lb.beam and b/deps/ibrowse/ebin/ibrowse_lb.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_lib.beam b/deps/ibrowse/ebin/ibrowse_lib.beam index ffe5172..1f9d905 100644 Binary files a/deps/ibrowse/ebin/ibrowse_lib.beam and b/deps/ibrowse/ebin/ibrowse_lib.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_sup.beam b/deps/ibrowse/ebin/ibrowse_sup.beam index cc71806..0f74506 100644 Binary files a/deps/ibrowse/ebin/ibrowse_sup.beam and b/deps/ibrowse/ebin/ibrowse_sup.beam differ diff --git a/deps/kdht/ebin/bencode.beam b/deps/kdht/ebin/bencode.beam index 2c371eb..5080e66 100644 Binary files a/deps/kdht/ebin/bencode.beam and b/deps/kdht/ebin/bencode.beam differ diff --git a/deps/kdht/ebin/bucket.beam b/deps/kdht/ebin/bucket.beam index 9a63aaf..5189cfc 100644 Binary files a/deps/kdht/ebin/bucket.beam and b/deps/kdht/ebin/bucket.beam differ diff --git a/deps/kdht/ebin/dht_id.beam b/deps/kdht/ebin/dht_id.beam index 775ab7d..5f7e578 100644 Binary files a/deps/kdht/ebin/dht_id.beam and b/deps/kdht/ebin/dht_id.beam differ diff --git a/deps/kdht/ebin/dht_net.beam b/deps/kdht/ebin/dht_net.beam index 5c58ed5..aff964a 100644 Binary files a/deps/kdht/ebin/dht_net.beam and b/deps/kdht/ebin/dht_net.beam differ diff --git a/deps/kdht/ebin/dht_state.beam b/deps/kdht/ebin/dht_state.beam index 2cd8b01..1b115a2 100644 Binary files a/deps/kdht/ebin/dht_state.beam and b/deps/kdht/ebin/dht_state.beam differ diff --git a/deps/kdht/ebin/kdht_sup.beam b/deps/kdht/ebin/kdht_sup.beam index d38eb00..5f3ec2c 100644 Binary files a/deps/kdht/ebin/kdht_sup.beam and b/deps/kdht/ebin/kdht_sup.beam differ diff --git a/deps/kdht/ebin/msg.beam b/deps/kdht/ebin/msg.beam index 181a136..890f988 100644 Binary files a/deps/kdht/ebin/msg.beam and b/deps/kdht/ebin/msg.beam differ diff --git a/deps/kdht/ebin/search.beam b/deps/kdht/ebin/search.beam index 82a2c68..f630f7f 100644 Binary files a/deps/kdht/ebin/search.beam and b/deps/kdht/ebin/search.beam differ diff --git a/deps/kdht/ebin/storage.beam b/deps/kdht/ebin/storage.beam index 0d2b0bc..cf7229b 100644 Binary files a/deps/kdht/ebin/storage.beam and b/deps/kdht/ebin/storage.beam differ diff --git a/deps/kdht/ebin/timer_monitor.beam b/deps/kdht/ebin/timer_monitor.beam index 0d19e07..2369cde 100644 Binary files a/deps/kdht/ebin/timer_monitor.beam and b/deps/kdht/ebin/timer_monitor.beam differ diff --git a/deps/kdht/ebin/vlog.beam b/deps/kdht/ebin/vlog.beam index 3e7c5d9..14706c9 100644 Binary files a/deps/kdht/ebin/vlog.beam and b/deps/kdht/ebin/vlog.beam differ diff --git a/deps/mongodb/ebin/mongo.beam b/deps/mongodb/ebin/mongo.beam index 536ba6c..b7c4d61 100644 Binary files a/deps/mongodb/ebin/mongo.beam and b/deps/mongodb/ebin/mongo.beam differ diff --git a/deps/mongodb/ebin/mongo_app.beam b/deps/mongodb/ebin/mongo_app.beam index 8a13224..190225e 100644 Binary files a/deps/mongodb/ebin/mongo_app.beam and b/deps/mongodb/ebin/mongo_app.beam differ diff --git a/deps/mongodb/ebin/mongo_connection.beam b/deps/mongodb/ebin/mongo_connection.beam index 10242bd..982951d 100644 Binary files a/deps/mongodb/ebin/mongo_connection.beam and b/deps/mongodb/ebin/mongo_connection.beam differ diff --git a/deps/mongodb/ebin/mongo_cursor.beam b/deps/mongodb/ebin/mongo_cursor.beam index d2baa54..58cf89e 100644 Binary files a/deps/mongodb/ebin/mongo_cursor.beam and b/deps/mongodb/ebin/mongo_cursor.beam differ diff --git a/deps/mongodb/ebin/mongo_id_server.beam b/deps/mongodb/ebin/mongo_id_server.beam index 1e8bb6e..ef86195 100644 Binary files a/deps/mongodb/ebin/mongo_id_server.beam and b/deps/mongodb/ebin/mongo_id_server.beam differ diff --git a/deps/mongodb/ebin/mongo_pool.beam b/deps/mongodb/ebin/mongo_pool.beam index 9e3841e..d4a7d6d 100644 Binary files a/deps/mongodb/ebin/mongo_pool.beam and b/deps/mongodb/ebin/mongo_pool.beam differ diff --git a/deps/mongodb/ebin/mongo_protocol.beam b/deps/mongodb/ebin/mongo_protocol.beam index e96fdb4..3ecf7b7 100644 Binary files a/deps/mongodb/ebin/mongo_protocol.beam and b/deps/mongodb/ebin/mongo_protocol.beam differ diff --git a/deps/mongodb/ebin/mongo_sup.beam b/deps/mongodb/ebin/mongo_sup.beam index be18196..ce8c0e7 100644 Binary files a/deps/mongodb/ebin/mongo_sup.beam and b/deps/mongodb/ebin/mongo_sup.beam differ diff --git a/ebin/bt_conn.beam b/ebin/bt_conn.beam index f07aa66..b6b9010 100644 Binary files a/ebin/bt_conn.beam and b/ebin/bt_conn.beam differ diff --git a/ebin/bt_message.beam b/ebin/bt_message.beam index 74bb87a..8fd1b8c 100644 Binary files a/ebin/bt_message.beam and b/ebin/bt_message.beam differ diff --git a/ebin/config.beam b/ebin/config.beam index 7f8c6f3..8d93e8b 100644 Binary files a/ebin/config.beam and b/ebin/config.beam differ diff --git a/ebin/crawler_app.beam b/ebin/crawler_app.beam index 13d4067..83f53cc 100644 Binary files a/ebin/crawler_app.beam and b/ebin/crawler_app.beam differ diff --git a/ebin/crawler_http.beam b/ebin/crawler_http.beam index ab9ca4d..09f33d9 100644 Binary files a/ebin/crawler_http.beam and b/ebin/crawler_http.beam differ diff --git a/ebin/crawler_stats.beam b/ebin/crawler_stats.beam index e8ba8be..2c10bb5 100644 Binary files a/ebin/crawler_stats.beam and b/ebin/crawler_stats.beam differ diff --git a/ebin/crawler_sup.beam b/ebin/crawler_sup.beam index a45b35b..4609f59 100644 Binary files a/ebin/crawler_sup.beam and b/ebin/crawler_sup.beam differ diff --git a/ebin/db_frontend.beam b/ebin/db_frontend.beam index 8512f2a..3de907a 100644 Binary files a/ebin/db_frontend.beam and b/ebin/db_frontend.beam differ diff --git a/ebin/db_hash.beam b/ebin/db_hash.beam index 1c38372..fb2703c 100644 Binary files a/ebin/db_hash.beam and b/ebin/db_hash.beam differ diff --git a/ebin/db_hash_reader.beam b/ebin/db_hash_reader.beam deleted file mode 100644 index 2a0a7f6..0000000 Binary files a/ebin/db_hash_reader.beam and /dev/null differ diff --git a/ebin/db_hash_reader_sup.beam b/ebin/db_hash_reader_sup.beam deleted file mode 100644 index b5edf76..0000000 Binary files a/ebin/db_hash_reader_sup.beam and /dev/null differ diff --git a/ebin/db_loc_torrent.beam b/ebin/db_loc_torrent.beam index b51b8c5..38a9a43 100644 Binary files a/ebin/db_loc_torrent.beam and b/ebin/db_loc_torrent.beam differ diff --git a/ebin/db_store_mongo.beam b/ebin/db_store_mongo.beam index cb8409d..373080c 100644 Binary files a/ebin/db_store_mongo.beam and b/ebin/db_store_mongo.beam differ diff --git a/ebin/db_system.beam b/ebin/db_system.beam index acf6a68..1da2778 100644 Binary files a/ebin/db_system.beam and b/ebin/db_system.beam differ diff --git a/ebin/dht_monitor.beam b/ebin/dht_monitor.beam index 6f9547d..75b8afb 100644 Binary files a/ebin/dht_monitor.beam and b/ebin/dht_monitor.beam differ diff --git a/ebin/dhtcrawler.app b/ebin/dhtcrawler.app index 08412b8..1fc9d0e 100644 --- a/ebin/dhtcrawler.app +++ b/ebin/dhtcrawler.app @@ -1,16 +1,16 @@ {application,dhtcrawler, [{description,"A DHT crawler to index magnet hash to torrent"}, - {vsn,"5d8c198"}, + {vsn,"b9130ec"}, {registered,[dht_crawler_sup]}, {applications,[kernel,stdlib,crypto,public_key,ssl,inets,bson, mongodb]}, {mod,{crawler_app,[]}}, {modules,[bt_conn,bt_message,config,crawler_app,crawler_http, crawler_stats,crawler_sup,db_frontend,db_hash, - db_hash_reader,db_hash_reader_sup,db_loc_torrent, - db_store_mongo,db_system,dht_monitor, - hash_reader_stats,http_cache,http_handler,loc_torrent, - loc_torrent_cache,loc_torrent_sup,string_split, - time_util,tor_builder,tor_download,tor_download_stats, + db_loc_torrent,db_store_mongo,db_system,dht_monitor, + hash_reader,hash_reader_stats,hash_reader_sup, + http_cache,http_handler,loc_torrent,loc_torrent_cache, + loc_torrent_sup,string_split,time_util,tor_builder, + tor_download,tor_download_stats,tor_location, tor_location_reader,tor_location_writer,torrent_file, - urldecode,vlog]}]}. + transfer,urldecode,vlog]}]}. diff --git a/ebin/hash_reader.beam b/ebin/hash_reader.beam new file mode 100644 index 0000000..eea33e0 Binary files /dev/null and b/ebin/hash_reader.beam differ diff --git a/ebin/hash_reader_stats.beam b/ebin/hash_reader_stats.beam index c0186a7..ec7d6ce 100644 Binary files a/ebin/hash_reader_stats.beam and b/ebin/hash_reader_stats.beam differ diff --git a/ebin/hash_reader_sup.beam b/ebin/hash_reader_sup.beam new file mode 100644 index 0000000..ef47211 Binary files /dev/null and b/ebin/hash_reader_sup.beam differ diff --git a/ebin/http_cache.beam b/ebin/http_cache.beam index 38b8a8d..071c154 100644 Binary files a/ebin/http_cache.beam and b/ebin/http_cache.beam differ diff --git a/ebin/http_handler.beam b/ebin/http_handler.beam index ad24b7d..b32582b 100644 Binary files a/ebin/http_handler.beam and b/ebin/http_handler.beam differ diff --git a/ebin/loc_torrent.beam b/ebin/loc_torrent.beam index ce42375..f3d3767 100644 Binary files a/ebin/loc_torrent.beam and b/ebin/loc_torrent.beam differ diff --git a/ebin/loc_torrent_cache.beam b/ebin/loc_torrent_cache.beam index add771c..813c051 100644 Binary files a/ebin/loc_torrent_cache.beam and b/ebin/loc_torrent_cache.beam differ diff --git a/ebin/loc_torrent_sup.beam b/ebin/loc_torrent_sup.beam index e4fc1f3..1207ef5 100644 Binary files a/ebin/loc_torrent_sup.beam and b/ebin/loc_torrent_sup.beam differ diff --git a/ebin/string_split.beam b/ebin/string_split.beam index daf9506..5ed8fe5 100644 Binary files a/ebin/string_split.beam and b/ebin/string_split.beam differ diff --git a/ebin/time_util.beam b/ebin/time_util.beam index efb3139..bced3e7 100644 Binary files a/ebin/time_util.beam and b/ebin/time_util.beam differ diff --git a/ebin/tor_builder.beam b/ebin/tor_builder.beam index faadb3f..8b51fff 100644 Binary files a/ebin/tor_builder.beam and b/ebin/tor_builder.beam differ diff --git a/ebin/tor_download.beam b/ebin/tor_download.beam index b0c301f..3e633f3 100644 Binary files a/ebin/tor_download.beam and b/ebin/tor_download.beam differ diff --git a/ebin/tor_download_stats.beam b/ebin/tor_download_stats.beam index 7756ce8..02d3675 100644 Binary files a/ebin/tor_download_stats.beam and b/ebin/tor_download_stats.beam differ diff --git a/ebin/tor_location_reader.beam b/ebin/tor_location_reader.beam index 457859f..c40c4bb 100644 Binary files a/ebin/tor_location_reader.beam and b/ebin/tor_location_reader.beam differ diff --git a/ebin/tor_location_writer.beam b/ebin/tor_location_writer.beam index 180f0c3..71bc78e 100644 Binary files a/ebin/tor_location_writer.beam and b/ebin/tor_location_writer.beam differ diff --git a/ebin/torrent_file.beam b/ebin/torrent_file.beam index 75b0dea..9fad45a 100644 Binary files a/ebin/torrent_file.beam and b/ebin/torrent_file.beam differ diff --git a/ebin/urldecode.beam b/ebin/urldecode.beam index 24d3c9e..9d0f13d 100644 Binary files a/ebin/urldecode.beam and b/ebin/urldecode.beam differ diff --git a/ebin/vlog.beam b/ebin/vlog.beam index b7a9db5..d67dbec 100644 Binary files a/ebin/vlog.beam and b/ebin/vlog.beam differ