rats-search/src/background/spider.js
2018-03-28 23:35:58 +03:00

797 lines
18 KiB
JavaScript

const config = require('./config');
const client = new (require('./bt/client'))
const spider = new (require('./bt/spider'))(client)
const fs = require('fs');
const mysql = require('mysql');
const getPeersStatisticUDP = require('./bt/udp-tracker-request')
const crypto = require('crypto')
const P2PServer = require('./p2p')
const stun = require('stun')
const natUpnp = require('nat-upnp');
const http = require('https')
const API = require('./api')
//var express = require('express');
//var app = express();
//var server = http.Server(app);
//var io = require('socket.io')(server);
//var sm = require('sitemap');
//var phantomjs = require('phantomjs-prebuilt')
//const disk = require('diskusage');
const encryptor = require('simple-encryptor')('rats-on-the-boat-enc-v0');
const os = require('os');
let rootPath = os.platform() === 'win32' ? 'c:' : '/';
const _debug = require('debug')
const cleanupDebug = _debug('main:cleanup');
const balanceDebug = _debug('main:balance');
const fakeTorrentsDebug = _debug('main:fakeTorrents');
const quotaDebug = _debug('main:quota');
const checkInternet = require('./checkInternet')
const {torrentTypeDetect} = require('../app/content');
const torrentClient = require('./torrentClient')
// Start server
//server.listen(config.httpPort);
//console.log('Listening web server on', config.httpPort, 'port')
module.exports = function (send, recive, dataDirectory, version)
{
let torrentsId = 1;
let filesId = 1;
let sphinx = mysql.createPool({
connectionLimit: config.sphinx.connectionLimit,
host : config.sphinx.host,
port : config.sphinx.port
});
const udpTrackers = [
{
host: 'tracker.coppersurfer.tk',
port: 6969
},
{
host: 'tracker.leechers-paradise.org',
port: 6969
},
{
host: 'tracker.opentrackr.org',
port: 1337
},
{
host: '9.rarbg.me',
port: 2710
}
]
let mysqlSingle;
function handleListenerDisconnect() {
mysqlSingle = mysql.createConnection({
host : config.sphinx.host,
port : config.sphinx.port
});
mysqlSingle.connect(function(mysqlError) {
if (mysqlError) {
console.error('error connecting: ' + mysqlError.stack);
return;
}
mysqlSingle.query("SELECT MAX(`id`) as mx from torrents", (err, rows) => {
if(err)
return
if(rows[0] && rows[0].mx >= 1)
torrentsId = rows[0].mx + 1;
})
mysqlSingle.query("SELECT MAX(`id`) as mx from files", (err, rows) => {
if(err)
return
if(rows[0] &&rows[0].mx >= 1)
filesId = rows[0].mx + 1;
})
});
mysqlSingle.on('error', function(err) {
console.log('db error', err);
if(err.code === 'PROTOCOL_CONNECTION_LOST') { // Connection to the MySQL server is usually
handleListenerDisconnect(); // lost due to either server restart, or a
} else { // connnection idle timeout (the wait_timeout
throw err; // server variable configures this)
}
});
const query = mysqlSingle.query;
mysqlSingle.query = (...args) => {
let callback, i;
for(i = 1; i < args.length; i++)
{
if(typeof args[i] == 'function')
{
callback = args[i];
break;
}
}
if(callback)
{
pushDatabaseBalance();
args[i] = (...a) => {
popDatabaseBalance();
callback(...a)
}
}
else if(args.length <= 2)
{
pushDatabaseBalance();
args.push(() => {
popDatabaseBalance();
});
}
query.apply(mysqlSingle, args)
}
mysqlSingle.insertValues = (table, values, callback) => {
let names = '';
let data = '';
for(const val in values)
{
names += '`' + val + '`,';
data += mysqlSingle.escape(values[val]) + ',';
}
names = names.slice(0, -1)
data = data.slice(0, -1)
let query = `INSERT INTO ${table}(${names}) VALUES(${data})`;
if(callback)
return mysqlSingle.query(query, (...responce) => callback(...responce))
else
return mysqlSingle.query(query)
}
}
handleListenerDisconnect();
/*
app.use(express.static('build', {index: false}));
app.get('/sitemap.xml', function(req, res) {
sphinx.query('SELECT count(*) as cnt FROM `torrents` WHERE contentCategory != \'xxx\' OR contentCategory IS NULL', function (error, rows, fields) {
if(!rows) {
return;
}
let urls = []
for(let i = 0; i < Math.ceil(rows[0].cnt / config.sitemapMaxSize); i++)
urls.push(`http://${config.domain}/sitemap${i+1}.xml`);
res.header('Content-Type', 'application/xml');
res.send( sm.buildSitemapIndex({
urls
}));
});
});
app.get('/sitemap:id.xml', function(req, res) {
if(req.params.id < 1)
return;
let page = (req.params.id - 1) * config.sitemapMaxSize
sphinx.query('SELECT hash FROM `torrents` WHERE contentCategory != \'xxx\' OR contentCategory IS NULL LIMIT ?, ?', [page, config.sitemapMaxSize], function (error, rows, fields) {
if(!rows) {
return;
}
let sitemap = sm.createSitemap ({
hostname: 'http://' + config.domain,
cacheTime: 600000
});
sitemap.add({url: '/'});
for(let i = 0; i < rows.length; i++)
sitemap.add({url: '/torrent/' + rows[i].hash});
sitemap.toXML( function (err, xml) {
if (err) {
return res.status(500).end();
}
res.header('Content-Type', 'application/xml');
res.send( xml );
});
});
});
app.get('*', function(req, res)
{
if(typeof req.query['_escaped_fragment_'] != 'undefined')
{
let program = phantomjs.exec('phantom.js', 'http://' + config.domain + req.path)
let body = '';
let timeout = setTimeout(() => {
program.kill();
}, 45000)
program.stderr.pipe(process.stderr)
program.stdout.on('data', (chunk) => {
body += chunk;
});
program.on('exit', code => {
clearTimeout(timeout);
res.header('Content-Type', 'text/html');
res.send( body );
})
return;
}
res.sendfile(__dirname + '/build/index.html');
});
*/
// start
function baseRowData(row)
{
return {
hash: row.hash,
name: row.name,
size: row.size,
files: row.files,
filesList: row.filesList,
piecelength: row.piecelength,
added: row.added ? (typeof row.added === 'object' ? row.added.getTime() : row.added) : (new Date()).getTime(),
contentType: row.contentType || row.contenttype,
contentCategory: row.contentCategory || row.contentcategory,
seeders: row.seeders,
completed: row.completed,
leechers: row.leechers,
trackersChecked: row.trackersChecked ? row.trackersChecked.getTime() : undefined,
good: row.good,
bad: row.bad,
}
}
const p2p = new P2PServer(send)
p2p.version = version
p2p.encryptor = encryptor
p2p.listen()
// load initial peers
if(dataDirectory && fs.existsSync(dataDirectory + '/peers.p2p'))
{
const peersEncrypted = fs.readFileSync(dataDirectory + '/peers.p2p', 'utf8')
const peers = encryptor.decrypt(peersEncrypted)
if(peers && peers.length > 0)
{
peers.forEach(peer => p2p.add(peer))
console.log('loaded', peers.length, 'peers')
}
}
if(config.p2pBootstrap)
{
http.get('https://api.myjson.com/bins/1e5rmh', (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
const json = JSON.parse(data)
if(json.bootstrap)
{
const peers = encryptor.decrypt(json.bootstrap)
if(peers && peers.length > 0)
{
peers.forEach(peer => p2p.add(peer))
console.log('loaded', peers.length, 'peers from bootstrap')
}
}
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
}
let undoneQueries = 0;
let pushDatabaseBalance = () => {
undoneQueries++;
if(undoneQueries >= 5000)
{
balanceDebug('start balance mysql, queries:', undoneQueries);
spider.ignore = true;
}
};
let popDatabaseBalance = () => {
undoneQueries--;
balanceDebug('balanced, queries left:', undoneQueries);
if(undoneQueries == 0)
{
balanceDebug('balance done');
spider.ignore = false;
}
};
// обновление статистики
/*
setInterval(() => {
let stats = {};
sphinx.query('SELECT COUNT(*) as tornum FROM `torrents`', function (error, rows, fields) {
stats.torrents = rows[0].tornum;
sphinx.query('SELECT COUNT(*) as filesnum, SUM(`size`) as filesizes FROM `files`', function (error, rows, fields) {
stats.files = rows[0].filesnum;
stats.size = rows[0].filesizes;
send('newStatistic', stats);
sphinx.query('DELETE FROM `statistic`', function (err, result) {
if(!result) {
console.error(err);
}
sphinx.query('INSERT INTO `statistic` SET ?', stats, function(err, result) {
if(!result) {
console.error(err);
}
});
})
});
});
}, 10 * 60 * 1000)
*/
const updateTorrentTrackers = (hash) => {
let maxSeeders = 0, maxLeechers = 0, maxCompleted = 0;
mysqlSingle.query('UPDATE torrents SET trackersChecked = ? WHERE hash = ?', [Math.floor(Date.now() / 1000), hash], function(err, result) {
if(!result) {
console.error(err);
return
}
udpTrackers.forEach((tracker) => {
getPeersStatisticUDP(tracker.host, tracker.port, hash, ({seeders, completed, leechers}) => {
if(seeders == 0 && completed == 0 && leechers == 0)
return;
if(seeders < maxSeeders)
{
return;
}
if(seeders == maxSeeders && leechers < maxLeechers)
{
return;
}
if(seeders == maxSeeders && leechers == maxLeechers && completed <= maxCompleted)
{
return;
}
maxSeeders = seeders;
maxLeechers = leechers;
maxCompleted = completed;
let checkTime = new Date();
mysqlSingle.query('UPDATE torrents SET seeders = ?, completed = ?, leechers = ?, trackersChecked = ? WHERE hash = ?', [seeders, completed, leechers, Math.floor(checkTime.getTime() / 1000), hash], function(err, result) {
if(!result) {
console.error(err);
return
}
send('trackerTorrentUpdate', {
hash,
seeders,
completed,
leechers,
trackersChecked: checkTime.getTime()
});
});
});
});
});
}
const cleanupTorrents = (cleanTorrents = 1) => {
if(!config.cleanup)
return;
/*
disk.check(rootPath, function(err, info) {
if (err) {
console.log(err);
} else {
const {available, free, total} = info;
if(free < config.cleanupDiscLimit)
{
mysqlSingle.query(`SELECT * FROM torrents WHERE added < DATE_SUB(NOW(), INTERVAL 6 hour) ORDER BY seeders ASC, files DESC, leechers ASC, completed ASC LIMIT ${cleanTorrents}`, function(err, torrents) {
if(!torrents)
return;
torrents.forEach((torrent) => {
if(torrent.seeders > 0){
cleanupDebug('this torrent is ok', torrent.name);
return
}
cleanupDebug('cleanup torrent', torrent.name, '[seeders', torrent.seeders, ', files', torrent.files, ']', 'free', (free / (1024 * 1024)) + "mb");
mysqlSingle.query('DELETE FROM files WHERE hash = ?', torrent.hash);
mysqlSingle.query('DELETE FROM torrents WHERE hash = ?', torrent.hash);
})
});
}
else
cleanupDebug('enough free space', (free / (1024 * 1024)) + "mb");
}
});
*/
}
const insertTorrentToDB = (torrent) => {
if(!torrent)
return
const { filesList } = torrent
delete torrent.filesList;
torrent.id = torrentsId++;
mysqlSingle.query("SELECT id FROM torrents WHERE hash = ?", torrent.hash, (err, single) => {
if(!single)
{
console.log(err)
return
}
if(single.length > 0)
{
return
}
mysqlSingle.insertValues('torrents', torrent, function(err, result) {
if(result) {
send('newTorrent', {
hash: torrent.hash,
name: torrent.name,
size: torrent.size,
files: torrent.files,
piecelength: torrent.piecelength,
contentType: torrent.contentType,
contentCategory: torrent.contentCategory,
});
updateTorrentTrackers(torrent.hash);
}
else
{
console.log(torrent);
console.error(err);
}
});
})
let filesToAdd = filesList.length;
mysqlSingle.query('SELECT count(*) as files_count FROM files WHERE hash = ?', [torrent.hash], function(err, rows) {
if(!rows)
return
const db_files = rows[0]['files_count'];
if(db_files !== torrent.files)
{
mysqlSingle.query('DELETE FROM files WHERE hash = ?', torrent.hash, function (err, result) {
if(err)
{
return;
}
filesList.forEach((file) => {
file.id = filesId++;
mysqlSingle.insertValues('files', file, function(err, result) {
if(!result) {
console.log(file);
console.error(err);
return
}
if(--filesToAdd === 0) {
send('filesReady', torrent.hash);
}
});
});
})
}
})
}
const updateTorrent = (metadata, infohash, rinfo) => {
console.log('finded torrent', metadata.info.name, ' and add to database');
const hash = infohash.toString('hex');
let size = metadata.info.length ? metadata.info.length : 0;
let filesCount = 1;
let filesArray = [];
const filesAdd = (path, size) => filesArray.push({
hash,
path,
pathIndex: path,
size,
})
if(metadata.info.files && metadata.info.files.length > 0)
{
filesCount = metadata.info.files.length;
size = 0;
for(let i = 0; i < metadata.info.files.length; i++)
{
let file = metadata.info.files[i];
let filePath = file.path.join('/');
filesAdd(filePath, file.length);
size += file.length;
}
}
else
{
filesAdd(metadata.info.name, size)
}
const torrentQ = {
hash: hash,
name: metadata.info.name,
nameIndex: metadata.info.name,
size: size,
files: filesCount,
piecelength: metadata.info['piece length'],
ipv4: rinfo.address,
port: rinfo.port,
added: Math.floor(Date.now() / 1000)
};
torrentTypeDetect(torrentQ, filesArray);
torrentQ.filesList = filesArray;
insertTorrentToDB(torrentQ)
}
client.on('complete', function (metadata, infohash, rinfo) {
cleanupTorrents(1); // clean old torrents before writing new
if(config.spaceQuota && config.spaceDiskLimit > 0)
{
disk.check(rootPath, function(err, info) {
if (err) {
console.log(err);
} else {
const {available, free, total} = info;
if(free >= config.spaceDiskLimit)
{
hideFakeTorrents(); // also enable fake torrents;
updateTorrent(metadata, infohash, rinfo);
}
else
{
quotaDebug('ignore torrent', metadata.info.name, 'free space', (free / (1024 * 1024)) + "mb");
showFakeTorrents(); // also enable fake torrents;
}
}
});
}
else
{
updateTorrent(metadata, infohash, rinfo);
}
});
checkInternet((connected) => {
if(!connected)
return
const { STUN_BINDING_REQUEST, STUN_ATTR_XOR_MAPPED_ADDRESS } = stun.constants
const stunServer = stun.createServer()
const stunRequest = stun.createMessage(STUN_BINDING_REQUEST)
stunServer.once('bindingResponse', stunMsg => {
const {address, port} = stunMsg.getAttribute(STUN_ATTR_XOR_MAPPED_ADDRESS).value
stunServer.close()
console.log('p2p stun ignore my address', address)
p2p.ignore(address)
// check port avalibility
p2p.checkPortAndRedirect(address, config.spiderPort)
})
stunServer.send(stunRequest, 19302, 'stun.l.google.com')
})
let upnp
if(config.upnp)
{
upnp = natUpnp.createClient();
upnp.ratsMap = () => {
upnp.portMapping({
public: config.spiderPort,
private: config.spiderPort,
protocol: 'UDP',
description: 'Rats',
ttl: 0
}, function(err) {
if(err)
console.log('upnp server dont respond')
});
upnp.portMapping({
public: config.spiderPort,
private: config.spiderPort,
protocol: 'TCP',
description: 'Rats',
ttl: 0
}, function(err) {
if(err)
console.log('upnp server dont respond')
});
upnp.portMapping({
public: config.udpTrackersPort,
private: config.udpTrackersPort,
protocol: 'UDP',
description: 'Rats',
ttl: 0
}, function(err) {
if(err)
console.log('upnp server dont respond')
});
}
upnp.ratsUnmap = () => {
upnp.portUnmapping({
public: config.spiderPort,
protocol: 'UDP'
});
upnp.portUnmapping({
public: config.spiderPort,
protocol: 'TCP'
});
upnp.portUnmapping({
public: config.udpTrackersPort,
protocol: 'UDP'
});
}
upnp.ratsMap();
upnp.externalIp(function(err, ip) {
if(err)
return
console.log('p2p upnp ignore my address', ip)
p2p.ignore(ip)
});
}
spider.on('peer', (IPs) => {
IPs.forEach(ip => p2p.add(ip))
})
// setup api
API({
sphinx,
recive,
send,
p2p,
config,
baseRowData,
torrentClient,
spider,
upnp,
crypto
})
let fakeTorrents = [];
function showFakeTorrentsPage(page)
{
mysqlSingle.query('SELECT * FROM torrents LIMIT ?, 100', [page], function(err, torrents) {
if(!torrents)
return;
torrents.forEach((torrent, index) => {
const fk = fakeTorrents.push(setTimeout(() => {
delete fakeTorrents[fk-1];
send('newTorrent', baseRowData(torrent));
updateTorrentTrackers(torrent.hash);
fakeTorrentsDebug('fake torrent', torrents.name, 'index, page:', index, page);
}, 700 * index))
})
const fk = fakeTorrents.push(setTimeout(()=>{
delete fakeTorrents[fk-1];
showFakeTorrentsPage(torrents.length > 0 ? page + torrents.length : 0)
}, 700 * torrents.length))
});
}
function showFakeTorrents()
{
fakeTorrentsDebug('showing fake torrents');
hideFakeTorrents()
showFakeTorrentsPage(0);
}
function hideFakeTorrents()
{
fakeTorrents.forEach((fk) => {
clearTimeout(fk)
})
fakeTorrents = []
fakeTorrentsDebug('hidding fake torrents');
}
if(config.indexer) {
spider.listen(config.spiderPort)
if(config.p2p)
{
spider.announceHashes = [crypto.createHash('sha1').update('degrats-v1').digest()]
}
} else {
// showFakeTorrents();
}
if(config.cleanup && config.indexer)
{
cleanupDebug('cleanup enabled');
cleanupDebug('cleanup disc limit', (config.cleanupDiscLimit / (1024 * 1024)) + 'mb');
}
if(config.spaceQuota)
{
quotaDebug('disk quota enabled');
}
this.stop = (callback) => {
console.log('spider closing...')
if(upnp)
upnp.ratsUnmap()
console.log('closing p2p...')
p2p.close()
const close = () => {
torrentClient.destroy(() => {
sphinx.end(() => spider.close(() => {
mysqlSingle.destroy()
console.log('spider closed')
callback()
}))
})
}
// safe future peers
if(dataDirectory)
{
const addresses = p2p.addresses(p2p.peersList())
const peersEncripted = encryptor.encrypt(addresses)
fs.writeFileSync(dataDirectory + '/peers.p2p', peersEncripted, 'utf8');
console.log('peers saved')
if(config.p2pBootstrap && addresses.length > 5)
{
const options = {
port: 443,
host: 'api.myjson.com',
method: 'PUT',
path: '/bins/1e5rmh',
headers: {
'Content-Type' : "application/json",
}
};
console.log('save bootstrap peers')
const req = http.request(options, close);
req.on('error', close)
req.end(JSON.stringify({bootstrap: peersEncripted}))
}
else
{
close()
}
}
else
{
close()
}
}
return this
}