evan/rats-search
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'opt'

Browse Source
This commit is contained in:
Alexey Kasyanchuk
2018-03-01 17:46:35 +03:00
parent 102aac7b54 1d9cb84a18
commit aa9a32fd4e
5 changed files with 234 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
src/app/admin-page.js
View File

@ -134,24 +134,6 @@ export default class AdminPage extends Page {
}} />
</div>
<div className='row inline w100p'>
<div style={{flex: 1}}>CPU usage limitation</div>
<Slider
min={0}
max={100}
step={1}
style={{width: 300}}
value={this.options.spider && this.options.spider.cpuLimit === 0 ? 100 : this.options.spider && this.options.spider.cpuLimit}
onChange={(event, value) => {
if(value === 100)
value = 0
this.options.spider.cpuLimit = value
this.forceUpdate()
}}
/>
</div>
<div className='row inline w100p'>
<div style={{flex: 1}}>Download torrents directory</div>
<TextField
@ -175,6 +157,64 @@ export default class AdminPage extends Page {
}} />
</div>
<div style={{marginTop: 10}}>Torrent network scanner settings:</div>
<div className='column w100p'>
<div className='row inline w100p'>
<div style={{flex: 1}}>Scanner walk speed (current: {this.options.spider && this.options.spider.walkInterval}) [affected after program reload]</div>
<Slider
min={1}
max={150}
step={1}
style={{width: 300}}
value={this.options.spider && this.options.spider.walkInterval}
onChange={(event, value) => {
this.options.spider.walkInterval = value
this.forceUpdate()
}}
/>
</div>
<div className='fs0-75' style={{color: 'grey'}}>* Low value - fast initial scanning and high cpu usage. High Value - low cpu usage but very slow scanning.
Good value between 3-60. Defaul value: 5</div>
</div>
<div className='column w100p'>
<div className='row inline w100p'>
<div style={{flex: 1}}>Nodes usage (current: {this.options.spider && this.options.spider.nodesUsage})</div>
<Slider
min={0}
max={1000}
step={1}
style={{width: 300}}
value={this.options.spider && this.options.spider.nodesUsage}
onChange={(event, value) => {
this.options.spider.nodesUsage = value
this.forceUpdate()
}}
/>
</div>
<div className='fs0-75' style={{color: 'grey'}}>* Low Value - very low usage of nodes, low network traffic, slow torrent scanning. High value - high traffic, fast scanning, high routers usage.
Recomended value between 10-1000. Defaul value: 100. 0 - Ignore this option (no limit).
</div>
</div>
<div className='column w100p'>
<div className='row inline w100p'>
<div style={{flex: 1}}>Reduce network packages (current: {this.options.spider && this.options.spider.packagesLimit})</div>
<Slider
min={0}
max={2000}
step={1}
style={{width: 300}}
value={this.options.spider && this.options.spider.packagesLimit}
onChange={(event, value) => {
this.options.spider.packagesLimit = value
this.forceUpdate()
}}
/>
</div>
<div className='fs0-75' style={{color: 'grey'}}>* Low Value - ignore more usless network packages, lower traffic and routers usage. High Value - high traffic and router usage in prospectю
Recomended value between 300-2000. Defaul value: 500. 0 - Ignore this option (no limit).
</div>
</div>
{
this.settingsSavedMessage
&&

109
src/background/bt/spider.js
View File

@ -5,14 +5,9 @@ const Emiter = require('events')
const bencode = require('bencode')
const {Table, Node} = require('./table')
const Token = require('./token')
const cpuUsage = require('./cpu-usage')
const config = require('../config')
const fs = require('fs')
const _debug = require('debug')
const cpuDebug = _debug('spider:cpu')
const trafficDebug = _debug('spider:traffic')
const bootstraps = [{
address: 'router.bittorrent.com',
port: 6881
@ -45,11 +40,14 @@ class Spider extends Emiter {
this.client = client
this.ignore = false; // ignore all requests
this.initialized = false;
this.trafficSpeed = 0
this.walkInterval = config.spider.walkInterval;
this.cpuLimit = config.spider.cpuLimit;
this.cpuInterval = config.spider.cpuInterval;
this.foundSpeed = 0;
this.foundCounter = 0;
setInterval(() => {
this.foundSpeed = this.foundCounter;
this.foundCounter = 0;
}, 1000)
this.announceHashes = []
}
@ -72,7 +70,7 @@ class Spider extends Emiter {
this.send(message, address)
}
getPeersRequest(infoHash) {
getPeersRequest(infoHash, address) {
const message = {
t: generateTid(),
y: 'q',
@ -82,13 +80,7 @@ class Spider extends Emiter {
info_hash: infoHash
}
}
for(const address of this.table.nodes)
{
if(parseInt(Math.random() * 5) !== 1)
continue;
this.send(message, address)
}
this.send(message, address)
}
announcePeer(infoHash, token, address, port)
@ -119,14 +111,10 @@ class Spider extends Emiter {
return
if(!this.client || this.client.isIdle()) {
if(
!this.ignore
&& (this.cpuLimit <= 0 || cpuUsage() < this.cpuLimit + this.cpuInterval)
&& (config.trafficMax <= 0 || this.trafficSpeed == 0 || this.trafficSpeed < config.trafficMax)
)
if(!this.ignore)
{
const node = this.table.shift()
if (node) {
if (node && (config.spider.nodesUsage === 0 || parseInt(Math.random() * this.table.nodes.length / config.spider.nodesUsage) === 0)) {
this.findNode(Node.neighbor(node.id, this.table.id), {address: node.address, port: node.port})
}
}
@ -170,11 +158,8 @@ class Spider extends Emiter {
}
onFindNodeRequest(message, address) {
if(this.cpuLimit > 0 && cpuUsage() > this.cpuLimit) {
return
}
if(config.trafficIgnoreDHT && config.trafficMax > 0 && this.trafficSpeed > 0 && this.trafficSpeed > config.trafficMax) {
if(config.spider.packagesLimit !== 0 && this.foundSpeed > config.spider.packagesLimit)
{
return
}
@ -192,14 +177,17 @@ class Spider extends Emiter {
nodes: Node.encodeNodes(this.table.first())
}
}, address)
// also check hashes of alive ones
for(const hash of this.announceHashes)
{
this.getPeersRequest(hash, address)
}
}
onGetPeersRequest(message, address) {
if(this.cpuLimit > 0 && cpuUsage() > this.cpuLimit) {
return
}
if(config.trafficIgnoreDHT && config.trafficMax > 0 && this.trafficSpeed > 0 && this.trafficSpeed > config.trafficMax) {
if(config.spider.packagesLimit !== 0 && this.foundSpeed > config.spider.packagesLimit)
{
return
}
@ -220,6 +208,12 @@ class Spider extends Emiter {
}, address)
this.emit('unensureHash', infohash.toString('hex').toUpperCase())
// also check hashes of alive ones
for(const hash of this.announceHashes)
{
this.getPeersRequest(hash, address)
}
}
onAnnouncePeerRequest(message, address) {
@ -239,19 +233,13 @@ class Spider extends Emiter {
};
this.emit('ensureHash', infohash.toString('hex').toUpperCase(), addressPair)
if(this.client && !this.ignore) {
cpuDebug('cpu usage:' + cpuUsage())
if(this.cpuLimit <= 0 || cpuUsage() <= this.cpuLimit + this.cpuInterval) {
this.client.add(addressPair, infohash);
}
this.client.add(addressPair, infohash);
}
}
onPingRequest(message, address) {
if(this.cpuLimit > 0 && cpuUsage() > this.cpuLimit) {
return
}
if(config.trafficIgnoreDHT && config.trafficMax > 0 && this.trafficSpeed > 0 && this.trafficSpeed > config.trafficMax) {
if(config.spider.packagesLimit !== 0 && this.foundSpeed > config.spider.packagesLimit)
{
return
}
@ -263,11 +251,13 @@ class Spider extends Emiter {
const message = bencode.decode(data)
if (message.y.toString() == 'r') {
if(message.r.nodes) {
this.foundCounter++;
this.onFoundNodes(message.r.nodes, message.r.token, address)
} else if(message.r.values) {
this.onFoundPeers(message.r.values, message.r.token, address)
}
} else if (message.y.toString() == 'q') {
this.foundCounter++;
switch(message.q.toString()) {
case 'get_peers':
this.onGetPeersRequest(message, address)
@ -308,39 +298,6 @@ class Spider extends Emiter {
}, 3000)
this.join()
this.walk()
if(config.trafficMax > 0)
{
trafficDebug('inore dht traffic', config.trafficIgnoreDHT)
const path = `/sys/class/net/${config.trafficInterface}/statistics/rx_bytes`
if(fs.existsSync(path))
{
trafficDebug('limitation', config.trafficMax / 1024, 'kbps/s')
let traffic = 0
this.trafficInterval = setInterval(() => {
fs.readFile(path, (err, newTraffic) => {
if(err)
return
if(traffic === 0)
traffic = newTraffic
this.trafficSpeed = (newTraffic - traffic) / config.trafficUpdateTime
trafficDebug('traffic rx', this.trafficSpeed / 1024, 'kbps/s')
traffic = newTraffic
})
}, 1000 * config.trafficUpdateTime)
}
}
this.announceSearchInterval = setInterval(() => {
for(const hash of this.announceHashes)
{
this.getPeersRequest(hash)
}
}, 3000)
}
close(callback)
@ -351,10 +308,6 @@ class Spider extends Emiter {
return
}
clearInterval(this.joinInterval)
if(this.trafficInterval)
clearInterval(this.trafficInterval)
if(this.announceSearchInterval)
clearInterval(this.announceSearchInterval)
this.closing = true
this.udp.close(() => {
this.initialized = false

22
src/background/config.js
View File

@ -21,8 +21,8 @@ let config = {
spider: {
walkInterval: 5,
cpuLimit: 0,
cpuInterval: 10,
nodesUsage: 100,
packagesLimit: 500
},
downloader: {
@ -35,11 +35,6 @@ let config = {
spaceQuota: false,
spaceDiskLimit: 7 * 1024 * 1024 * 1024,
trafficInterface: 'enp2s0',
trafficMax: 0,
trafficUpdateTime: 3, //secs
trafficIgnoreDHT: false,
dbPath: '',
client: {
@ -81,7 +76,18 @@ config.load = () => {
const obj = JSON.parse(data);
for(let prop in obj)
{
config[prop] = obj[prop]
// объединяем объекты
if(typeof config[prop] === 'object' && typeof obj[prop] === 'object')
{
for(const subProp in obj[prop])
{
config[prop][subProp] = obj[prop][subProp]
}
}
else
{
config[prop] = obj[prop]
}
debug('rats.json:', prop, '=', obj[prop])
}
}

126
src/background/p2p.js Normal file
View File

@ -0,0 +1,126 @@
const config = require('./config');
const net = require('net')
const JsonSocket = require('json-socket')
class p2p {
peers = []
ignoreAddresses = []
messageHandlers = {}
size = 0
constructor(send = () => {})
{
this.send = send
this.tcpServer = net.createServer();
this.tcpServer.on('connection', (socket) => {
//console.log('p2p server connection', socket.remoteAddress)
socket = new JsonSocket(socket);
socket.on('error', (err) => {})
socket.on('message', (message) => {
if(message.type && this.messageHandlers[message.type])
{
this.messageHandlers[message.type](message.data, (data) => {
socket.sendMessage({
id: message.id,
data
});
})
}
});
})
}
listen() {
console.log('listen p2p on', config.spiderPort, 'port')
this.tcpServer.listen(config.spiderPort);
}
on(type, callback) {
this.messageHandlers[type] = callback
}
add(address) {
const { peers } = this
if(this.size > 10)
return;
if(address.port <= 1 || address.port > 65535)
return;
if(this.ignoreAddresses.includes(address.address))
return;
for(let peer of peers)
{
if(peer.address === address.address) {
peer.port = address.port;
return;
}
}
this.connect(address)
}
connect(address)
{
this.peers.push(address)
const socket = new JsonSocket(new net.Socket()); //Decorate a standard net.Socket with JsonSocket
socket.on('connect', () => { //Don't send until we're connected
// add to peers
this.size++;
this.send('peer', this.size)
console.log('new peer', address)
const callbacks = {}
socket.on('message', (message) => {
if(message.id && callbacks[message.id])
{
callbacks[message.id](message.data);
delete callbacks[message.id];
}
});
const emit = (type, data, callback) => {
const id = Math.random().toString(36).substring(5)
if(callback)
callbacks[id] = callback;
socket.sendMessage({
id,
type,
data
});
}
address.emit = emit
});
socket.on('close', () => {
const index = this.peers.indexOf(address);
if(index >= 0)
{
if(this.peers[index].emit) // only autorized peers
{
this.size--;
this.send('peer', this.size)
}
this.peers.splice(index, 1);
console.log('close peer connection', address)
}
})
socket.on('error', (err) => {})
socket.connect(address.port, address.address);
}
emit(type, data, callback)
{
for(const peer of this.peers)
{
if(peer.emit)
peer.emit(type, data, callback)
}
}
}
module.exports = p2p

118
src/background/spider.js
View File

@ -3,9 +3,8 @@ const client = new (require('./bt/client'))
const spider = new (require('./bt/spider'))(client)
const mysql = require('mysql');
const getPeersStatisticUDP = require('./bt/udp-tracker-request')
const net = require('net')
const JsonSocket = require('json-socket')
const crypto = require('crypto')
const P2PServer = require('./p2p')
const stun = require('stun')
//var express = require('express');
//var app = express();
@ -253,30 +252,8 @@ setInterval(() => {
topCache = {};
}, 24 * 60 * 60 * 1000);
// socket
const messageHandlers = {}
const onSocketMessage = (type, callback) => {
messageHandlers[type] = callback
}
const tcpServer = net.createServer();
tcpServer.listen(config.spiderPort);
tcpServer.on('connection', (socket) => {
socket = new JsonSocket(socket);
socket.on('error', (err) => {})
socket.on('message', (message) => {
if(message.type && messageHandlers[message.type])
{
messageHandlers[message.type](message.data, (data) => {
socket.sendMessage({
id: message.id,
data
});
})
}
});
})
const p2p = new P2PServer(send)
p2p.listen()
//io.on('connection', function(socket)
//{
@ -437,7 +414,7 @@ tcpServer.on('connection', (socket) => {
})
});
onSocketMessage('searchTorrent', ({text, navigation} = {}, callback) => {
p2p.on('searchTorrent', ({text, navigation} = {}, callback) => {
if(!text)
return;
@ -541,7 +518,7 @@ tcpServer.on('connection', (socket) => {
})
});
onSocketMessage('searchFiles', ({text, navigation} = {}, callback) => {
p2p.on('searchFiles', ({text, navigation} = {}, callback) => {
if(!text)
return;
@ -1046,86 +1023,6 @@ client.on('complete', function (metadata, infohash, rinfo) {
});
const p2p = {
peers: [],
ignoreAddresses: [],
add(address) {
const { peers } = this
if(peers.length > 10)
return;
if(address.port <= 1 || address.port > 65535)
return;
if(this.ignoreAddresses.includes(address.address))
return;
for(let peer of peers)
{
if(peer.address === address.address) {
peer.port = address.port;
return;
}
}
this.connect(address)
},
connect(address)
{
this.peers.push(address)
const socket = new JsonSocket(new net.Socket()); //Decorate a standard net.Socket with JsonSocket
socket.on('connect', () => { //Don't send until we're connected
// add to peers
send('peer', this.peers.length)
console.log('new peer', address)
const callbacks = {}
socket.on('message', (message) => {
if(message.id && callbacks[message.id])
{
callbacks[message.id](message.data);
delete callbacks[message.id];
}
});
const emit = (type, data, callback) => {
const id = Math.random().toString(36).substring(5)
if(callback)
callbacks[id] = callback;
socket.sendMessage({
id,
type,
data
});
}
address.emit = emit
});
socket.on('close', () => {
const index = this.peers.indexOf(address);
if(index >= 0)
{
this.peers.splice(index, 1);
console.log('close peer connection', address)
send('peer', this.peers.length)
}
})
socket.on('error', (err) => {})
socket.connect(address.port, address.address);
},
emit(type, data, callback)
{
for(const peer of this.peers)
{
if(peer.emit)
peer.emit(type, data, callback)
}
}
}
const { STUN_BINDING_REQUEST, STUN_ATTR_XOR_MAPPED_ADDRESS } = stun.constants
const stunServer = stun.createServer()
const stunRequest = stun.createMessage(STUN_BINDING_REQUEST)
@ -1139,11 +1036,6 @@ stunServer.once('bindingResponse', stunMsg => {
stunServer.send(stunRequest, 19302, 'stun.l.google.com')
spider.on('peer', (IPs) => {
const { peers } = p2p;
if(peers.length > 10)
return
IPs.forEach(ip => p2p.add(ip))
})