TPB scrape
Hi, have a good day!!!
TPB please refer to page
TPB scrape to take advantage of this article here?
:: http://scrape.thepiratebay.org/
Scrape all torrents from the open trackers at once. get this file.
The file is in the the following format:
[info_hash_base64]:[seeders]:[leechers]\n
The [info_hash_base64] is the binary info_hash base64 encoded.
[seeders] are number of seeders in base10 ascii
[leechers] are number of leechers in base10 ascii
There is one line per info_hash terminated by the \n character.
The file is updated every 3 minutes, please don't retrieve it more than every 3 minutes.
Please use this way of scraping if you scrape alot of torrents from our trackers,
this will reduce load on our trackers and at the same time give you all the stats at once.
It will also give you a simpler handling of your indexed torrents that uses our tracker.
Using this metod is higly recommended if you index more then 10.000 torrents using our trackers.
PHP/MySQL code example:
test.sql
PHP Code:
-- Example torrents table
-- last_scraped: unixtime format -- info_hash: stored in base64 encoding
CREATE TABLE `torrents` ( `id` int(10) unsigned NOT NULL auto_increment, `name` varchar(255) default NULL, `tracker` varchar(255) default NULL, `seeders` int(10) unsigned default NULL, `leechers` int(10) unsigned default NULL, `last_scraped` int(10) unsigned default NULL, `info_hash` varchar(40) default NULL, PRIMARY KEY (`id`), UNIQUE KEY `info_hash` (`info_hash`) );
--Some sample data: -- (just some current torrents on the tracker they might disapear)
INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('k37dQKfsLJ98NIm19v3lG32EWrw=',5+5,8+4,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('XdDGpJBm2eSbrwJOjifBjYKRG0E=',5+2,8+4,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('u5Y2QOgvN+Sux8O7elTmZSnV45I=',5+0,8+1,'http://tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('Lc9x9zMdGYRA0B9Oa7j0GC3FRM8=',5+0,8+2,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('ufZcoUP56DzOxhh4iZdYwWUxM9k=',5+6,8+9,'http://tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('a6S4USSJA8wq2mKsCBXv4eYt8SE=',5+3,8+10,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('Ho4+xsmm0SUi08lfqgj6QKUsG3Q=',5+0,8+5,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('XI0RCfsDPOEc4uE/VE67GJNdsEU=',5+8,8+6,'http://tv.tracker.prq.to/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('Y+DQ4oHQ5AKMF93n/DJb/kSqdWU=',5+1,8+1,'http://tv.tracker.prq.to/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('X98AZNOuPAbA1DRcHujovGzCJy0=',5+0,8+3,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('xuDNk5x5hsQJtvlXywkL6dCpfk8=',5+3,8+13,'http://open.tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('s+HjXfq7Kkx45gh6ifRp1tmHbxo=',5+1,8+2,'http://tpb.tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('/Jx5zoZfPE3P3Xi8YjdsoqHmliA=',5+9,8+25,'http://tpb.tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('MF2MXmNBOws1W+1XFi5jWnsW/u4=',5+5,8+3,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('/Jx5zoZfPE3P3Xi8YjdsoqHmlis=',5+3,8+0,'http://tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('NEt3Qy+WvEh2DSG0ik9pYLfgRJM=',5+10,8+3,'http://tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('tX03O1rr3ut6sL61/aZkoKSP9Ik=',5+2,8+6,'http://open.tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('bR4qXRcr4mrRlcdzJtOeRK6sVkw=',5+0,8+1,'http://tracker.prq.to/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('+lo4+vo4EOtmfSAhGnA5y5Vg9rs=',5+1,8+0,'http://tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('BQ2nGY+gjtS5DBdFsaF85jwPhS8=',5+1,8+5,'http://tracker.prq.to/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('VgbRqfwogEIGSwaBuAiWzBqSBuM=',5+0,8+4,'http://tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('k0tMHF1bRnkVKiWkxKkkWdsMzMg=',5+29,8+2,'http://tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('xvxizrMTO3CFVOAOY9FOxsGFMCQ=',5+0,8+1,'http://vip.tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('23Hn73X4OOOLX1MehQ4mKMoXbVU=',5+9,8+13,'http://tv.tracker.thepiratebay.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('wZkn9HfxW83bNrh0OVKHKPDjhAY=',5+1,8+3,'http://tracker.prq.to/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('ZqiFqXTRZ4rSE8xCNAPB5n8WKOA=',5+0,8+3,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('waE8U1Ggn4EHfRY/VhYbh9c1aRg=',5+3,8+4,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('oRQ32XCmNpqSDbuJRr5cqxuuIRY=',5+0,8+3,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('Vs/XPyW+berBeyXLhwZ2nesTLo4=',5+0,8+5,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('QMtgehc5Fc7mm/xtiD5iW1MoLJs=',5+1,8+2,'http://tracker.prq.to/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('Ok8lq9i5cKVQWp3AhBv5fPMY9q0=',5+6,8+6,'http://yetanother.tracker.org/announce.php'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('CFGSx5DYw+KKqiVbfgE3kChdhPE=',5+1,8+3,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('/MuHkhlGMozI8HTm+aoaNiG4HNk=',5+4,8+0,'http://yetanother.tracker.org/announce.php'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('LrwNc8m+qzrwC+me63BEwnlWzDQ=',5+6,8+6,'http://tracker.prq.to/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('FCCOqJMbM4tb1a3xBpTAkY/VF3o=',5+3,8+0,'http://yetanother.tracker.org/announce.php'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('tFn1pHUU7/3c2sWIETbyBfou6uY=',5+1,8+5,'http://yetanother.tracker.org/announce.php'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('6K8GwrYqvJtxPHgU1pGW7jqXkm0=',5+7,8+6,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('6/vG/71oAzrULIbjYRJQs+dC1Vc=',5+1,8+0,'http://yetanother.tracker.org/announce.php'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('KEyT5o+FSezV+XlOxBn2VnlWk9E=',5+0,8+11,'http://yetanother.tracker.org/announce.php'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('Q1antpizQSbdw1VgFcs1ClhyQD4=',5+1,8+3,'http://tracker.prq.to/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('esByCfSnnMc6VnKYfYm+hdsH5A8=',5+1,8+4,'http://tracker.prq.to/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('psn29ZsLXpXbZs1P+iXxa4ueUpc=',5+1,8+1,'http://yetanother.tracker.org/announce.php'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('r9w7ZTIY2eFVuC9hDR5+4EqdddI=',5+1,8+0,'http://other.tracker.org/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('XV2JK0rMS0iM/zDn8rKzLMNTh0g=',5+8,8+6,'http://yetanother.tracker.org/announce.php'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('4ZVlA2egK7IoREmKxfNoqIh9qXA=',5+11,8+29,'http://yetanother.tracker.org/announce.php'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('GmAkI7zF1MThqb+885qRB/7pGLs=',5+5,8+5,'http://yetanother.tracker.org/announce.php'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('T1btxBua9BBFa/Ls4hOy6NRrnyA=',5+0,8+2,'http://tracker.prq.to/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('fyzefMY8PhoJUagPJlaEK/v4CrY=',5+0,8+9,'http://tracker.prq.to/announce'); INSERT INTO torrents (info_hash,seeders,leechers,tracker) VALUES('Cb+CGrtxEKCZ/c+uQsA8iQv8pxA=',5+0,8+2,'http://tracker.prq.to/announce');
tpb.php
PHP Code:
<?
$mysql_host = "127.0.0.1"; $mysql_user = "root"; $mysql_pass = "502t"; $mysql_db = "ob"; $connection = @mysql_connect("$mysql_host","$mysql_user","$mysql_pass") or dberr(); $db = @mysql_select_db("$mysql_db", $connection) or die("Couldn't select database.");
// Get info_hash for all our torrents by trackers included in the bulk-scrape. $r=mysql_query("SELECT info_hash,seeders,leechers FROM torrents WHERE tracker LIKE '%thepiratebay.org/announce%' OR tracker LIKE '%tracker.prq.to/announce%'");
// Count total number of torrents we have that are tracked by trackers included in the bulk-scrape. $totaltorrents=mysql_num_rows($r);
while ($rs=mysql_fetch_assoc($r)) {
/* * Here is the place to recode the info_hash to however you stored it in your real database. * For example if you store it base64-encoded you would: * * $rs[info_hash]=base64_encode(urldecode($rs[info_hash])) */
$torrents[$rs[info_hash]][seeders]=$rs[seeders]; $torrents[$rs[info_hash]][leechers]=$rs[leechers]; } // Download the scrape data $fp=bzopen("http://scrape.thepiratebay.org/all.txt.bz2", "r");
// Read data from file. $scrape_data=""; while (!feof($fp)) { $scrape_data .= bzread($fp, 65536); } bzclose($fp);
$scrape_data_rows=explode("\n", $scrape_data);
// Save some memory. unset($scrape_data);
$updated=0; // count how many torrents we updated;
foreach ($scrape_data_rows as $rows) { $row=explode(":", $rows); // $row[0] is info_hash in base64 encoding // $row[1] is number of seeders // $row[2] is number of leechers
if (is_array($torrents[$row[0]])) { // We have the torrent in our database.
// if we already have the correct number of peers, no need to update the database. if (($torrents[$row[0]][seeders] != $row[1]) && ($torrents[$row[0]][leechers] != $row[2])) { $r=mysql_query("UPDATE torrents SET seeders=$row[1], leechers=$row[2], last_scraped=unix_timestamp() WHERE info_hash='$row[0]'"); $updated++; }
}
}
// Set seeders/leechers to 0 for torrents we have'nt seen for 2 days. $r=mysql_query("UPDATE torrents SET seeders=0, leechers=0 WHERE last_scraped < unix_timestamp() - (86400*2) AND (tracker LIKE '%thepiratebay.org/announce%' OR tracker LIKE '%tracker.prq.to/announce%')");
echo "We updated $updated of $totaltorrents in our database\n"; ?>
Thank you!
Last edited by cilcil; 28th August 2010 at 09:33.
Reason: code set
|