From 590671e1f5bdb826a5978701209094eacd1d9f3a Mon Sep 17 00:00:00 2001 From: Alexey Kasyanchuk Date: Thu, 1 Mar 2018 00:54:22 +0300 Subject: [PATCH 1/6] perf(network): network usage optimization --- src/background/bt/spider.js | 39 +++++++---- src/background/bt/table.js | 4 +- src/background/p2p.js | 126 ++++++++++++++++++++++++++++++++++++ src/background/spider.js | 118 ++------------------------------- 4 files changed, 161 insertions(+), 126 deletions(-) create mode 100644 src/background/p2p.js diff --git a/src/background/bt/spider.js b/src/background/bt/spider.js index aa4a436..3ee96d5 100644 --- a/src/background/bt/spider.js +++ b/src/background/bt/spider.js @@ -52,6 +52,7 @@ class Spider extends Emiter { this.cpuInterval = config.spider.cpuInterval; this.announceHashes = [] + this.searchHashes = [] } send(message, address) { @@ -72,7 +73,7 @@ class Spider extends Emiter { this.send(message, address) } - getPeersRequest(infoHash) { + getPeersRequest(infoHash, address) { const message = { t: generateTid(), y: 'q', @@ -82,13 +83,7 @@ class Spider extends Emiter { info_hash: infoHash } } - for(const address of this.table.nodes) - { - if(parseInt(Math.random() * 5) !== 1) - continue; - - this.send(message, address) - } + this.send(message, address) } announcePeer(infoHash, token, address, port) @@ -126,7 +121,10 @@ class Spider extends Emiter { ) { const node = this.table.shift() - if (node) { + //console.log(parseInt(Math.random() * this.table.caption / (this.table.nodes.length || 0))) + //if (node) { + if (node && parseInt(Math.random() * this.table.nodes.length / 8) === 0) { + //console.log('walk', this.table.nodes.length) this.findNode(Node.neighbor(node.id, this.table.id), {address: node.address, port: node.port}) } } @@ -141,6 +139,7 @@ class Spider extends Emiter { this.table.add(node) } }) + //console.log('nodes', this.table.nodes.length) this.emit('nodes', nodes) // announce torrents @@ -154,6 +153,7 @@ class Spider extends Emiter { } onFoundPeers(peers, token, address) { + console.log('responce') if(token) { for(const hash of this.announceHashes) @@ -167,6 +167,7 @@ class Spider extends Emiter { const ips = Node.decodeCompactIP(peers) this.emit('peer', ips) + console.log('p', ips) } onFindNodeRequest(message, address) { @@ -192,6 +193,12 @@ class Spider extends Emiter { nodes: Node.encodeNodes(this.table.first()) } }, address) + + // also check hashes of alive ones + for(const hash of this.announceHashes) + { + this.getPeersRequest(hash, address) + } } onGetPeersRequest(message, address) { @@ -220,6 +227,12 @@ class Spider extends Emiter { }, address) this.emit('unensureHash', infohash.toString('hex').toUpperCase()) + + // also check hashes of alive ones + for(const hash of this.announceHashes) + { + this.getPeersRequest(hash, address) + } } onAnnouncePeerRequest(message, address) { @@ -241,7 +254,7 @@ class Spider extends Emiter { if(this.client && !this.ignore) { cpuDebug('cpu usage:' + cpuUsage()) if(this.cpuLimit <= 0 || cpuUsage() <= this.cpuLimit + this.cpuInterval) { - this.client.add(addressPair, infohash); + // this.client.add(addressPair, infohash); } } } @@ -335,12 +348,14 @@ class Spider extends Emiter { } } + /* this.announceSearchInterval = setInterval(() => { for(const hash of this.announceHashes) { this.getPeersRequest(hash) } }, 3000) + */ } close(callback) @@ -353,8 +368,8 @@ class Spider extends Emiter { clearInterval(this.joinInterval) if(this.trafficInterval) clearInterval(this.trafficInterval) - if(this.announceSearchInterval) - clearInterval(this.announceSearchInterval) + //if(this.announceSearchInterval) + // clearInterval(this.announceSearchInterval) this.closing = true this.udp.close(() => { this.initialized = false diff --git a/src/background/bt/table.js b/src/background/bt/table.js index 4cdebaa..5b0140b 100644 --- a/src/background/bt/table.js +++ b/src/background/bt/table.js @@ -57,9 +57,11 @@ class Table{ this.nodes = [] this.caption = cap } - add(node) { + add(node, onAdd) { if (this.nodes.length < this.caption) { this.nodes.push(node) + if(onAdd) + onAdd(node) } } shift() { diff --git a/src/background/p2p.js b/src/background/p2p.js new file mode 100644 index 0000000..ec22883 --- /dev/null +++ b/src/background/p2p.js @@ -0,0 +1,126 @@ +const config = require('./config'); +const net = require('net') +const JsonSocket = require('json-socket') + +class p2p { + peers = [] + ignoreAddresses = [] + messageHandlers = {} + size = 0 + + constructor(send = () => {}) + { + this.send = send + this.tcpServer = net.createServer(); + this.tcpServer.on('connection', (socket) => { + //console.log('p2p server connection', socket.remoteAddress) + socket = new JsonSocket(socket); + socket.on('error', (err) => {}) + socket.on('message', (message) => { + if(message.type && this.messageHandlers[message.type]) + { + this.messageHandlers[message.type](message.data, (data) => { + socket.sendMessage({ + id: message.id, + data + }); + }) + } + }); + }) + } + + listen() { + console.log('listen p2p on', config.spiderPort, 'port') + this.tcpServer.listen(config.spiderPort); + } + + on(type, callback) { + this.messageHandlers[type] = callback + } + + add(address) { + const { peers } = this + + if(this.size > 10) + return; + + if(address.port <= 1 || address.port > 65535) + return; + + if(this.ignoreAddresses.includes(address.address)) + return; + + for(let peer of peers) + { + if(peer.address === address.address) { + peer.port = address.port; + return; + } + } + this.connect(address) + } + + connect(address) + { + this.peers.push(address) + const socket = new JsonSocket(new net.Socket()); //Decorate a standard net.Socket with JsonSocket + socket.on('connect', () => { //Don't send until we're connected + // add to peers + this.size++; + this.send('peer', this.size) + console.log('new peer', address) + + const callbacks = {} + socket.on('message', (message) => { + if(message.id && callbacks[message.id]) + { + callbacks[message.id](message.data); + delete callbacks[message.id]; + } + }); + + const emit = (type, data, callback) => { + const id = Math.random().toString(36).substring(5) + if(callback) + callbacks[id] = callback; + socket.sendMessage({ + id, + type, + data + }); + } + address.emit = emit + }); + + socket.on('close', () => { + const index = this.peers.indexOf(address); + if(index >= 0) + { + if(this.peers[index].emit) // only autorized peers + { + this.size--; + this.send('peer', this.size) + } + this.peers.splice(index, 1); + + console.log('close peer connection', address) + } + }) + + socket.on('error', (err) => {}) + + socket.connect(address.port, address.address); + } + + emit(type, data, callback) + { + for(const peer of this.peers) + { + if(peer.emit) + peer.emit(type, data, callback) + } + } +} + +module.exports = p2p \ No newline at end of file diff --git a/src/background/spider.js b/src/background/spider.js index a85ba41..2e03840 100644 --- a/src/background/spider.js +++ b/src/background/spider.js @@ -3,9 +3,8 @@ const client = new (require('./bt/client')) const spider = new (require('./bt/spider'))(client) const mysql = require('mysql'); const getPeersStatisticUDP = require('./bt/udp-tracker-request') -const net = require('net') -const JsonSocket = require('json-socket') const crypto = require('crypto') +const P2PServer = require('./p2p') const stun = require('stun') //var express = require('express'); //var app = express(); @@ -253,30 +252,8 @@ setInterval(() => { topCache = {}; }, 24 * 60 * 60 * 1000); - - -// socket -const messageHandlers = {} -const onSocketMessage = (type, callback) => { - messageHandlers[type] = callback -} -const tcpServer = net.createServer(); -tcpServer.listen(config.spiderPort); -tcpServer.on('connection', (socket) => { - socket = new JsonSocket(socket); - socket.on('error', (err) => {}) - socket.on('message', (message) => { - if(message.type && messageHandlers[message.type]) - { - messageHandlers[message.type](message.data, (data) => { - socket.sendMessage({ - id: message.id, - data - }); - }) - } - }); -}) +const p2p = new P2PServer(send) +p2p.listen() //io.on('connection', function(socket) //{ @@ -437,7 +414,7 @@ tcpServer.on('connection', (socket) => { }) }); - onSocketMessage('searchTorrent', ({text, navigation} = {}, callback) => { + p2p.on('searchTorrent', ({text, navigation} = {}, callback) => { if(!text) return; @@ -541,7 +518,7 @@ tcpServer.on('connection', (socket) => { }) }); - onSocketMessage('searchFiles', ({text, navigation} = {}, callback) => { + p2p.on('searchFiles', ({text, navigation} = {}, callback) => { if(!text) return; @@ -1046,86 +1023,6 @@ client.on('complete', function (metadata, infohash, rinfo) { }); -const p2p = { - peers: [], - ignoreAddresses: [], - add(address) { - const { peers } = this - - if(peers.length > 10) - return; - - if(address.port <= 1 || address.port > 65535) - return; - - if(this.ignoreAddresses.includes(address.address)) - return; - - for(let peer of peers) - { - if(peer.address === address.address) { - peer.port = address.port; - return; - } - } - this.connect(address) - }, - connect(address) - { - this.peers.push(address) - const socket = new JsonSocket(new net.Socket()); //Decorate a standard net.Socket with JsonSocket - socket.on('connect', () => { //Don't send until we're connected - // add to peers - send('peer', this.peers.length) - console.log('new peer', address) - - const callbacks = {} - socket.on('message', (message) => { - if(message.id && callbacks[message.id]) - { - callbacks[message.id](message.data); - delete callbacks[message.id]; - } - }); - - const emit = (type, data, callback) => { - const id = Math.random().toString(36).substring(5) - if(callback) - callbacks[id] = callback; - socket.sendMessage({ - id, - type, - data - }); - } - address.emit = emit - }); - - socket.on('close', () => { - const index = this.peers.indexOf(address); - if(index >= 0) - { - this.peers.splice(index, 1); - - console.log('close peer connection', address) - send('peer', this.peers.length) - } - }) - - socket.on('error', (err) => {}) - - socket.connect(address.port, address.address); - }, - emit(type, data, callback) - { - for(const peer of this.peers) - { - if(peer.emit) - peer.emit(type, data, callback) - } - } -} - const { STUN_BINDING_REQUEST, STUN_ATTR_XOR_MAPPED_ADDRESS } = stun.constants const stunServer = stun.createServer() const stunRequest = stun.createMessage(STUN_BINDING_REQUEST) @@ -1139,11 +1036,6 @@ stunServer.once('bindingResponse', stunMsg => { stunServer.send(stunRequest, 19302, 'stun.l.google.com') spider.on('peer', (IPs) => { - const { peers } = p2p; - - if(peers.length > 10) - return - IPs.forEach(ip => p2p.add(ip)) }) From 0db324f53df7a4e9330d1e54eea05e1bf699cf88 Mon Sep 17 00:00:00 2001 From: Alexey Kasyanchuk Date: Thu, 1 Mar 2018 01:17:46 +0300 Subject: [PATCH 2/6] unused stuffs removed --- src/background/bt/table.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/background/bt/table.js b/src/background/bt/table.js index 5b0140b..4cdebaa 100644 --- a/src/background/bt/table.js +++ b/src/background/bt/table.js @@ -57,11 +57,9 @@ class Table{ this.nodes = [] this.caption = cap } - add(node, onAdd) { + add(node) { if (this.nodes.length < this.caption) { this.nodes.push(node) - if(onAdd) - onAdd(node) } } shift() { From bdaaa020cc23bd38d5fc565c0712e0dd214a585f Mon Sep 17 00:00:00 2001 From: Alexey Kasyanchuk Date: Thu, 1 Mar 2018 04:10:12 +0300 Subject: [PATCH 3/6] perf(network): limit package usage --- src/background/bt/spider.js | 82 ++++++++----------------------------- 1 file changed, 18 insertions(+), 64 deletions(-) diff --git a/src/background/bt/spider.js b/src/background/bt/spider.js index 3ee96d5..6c51104 100644 --- a/src/background/bt/spider.js +++ b/src/background/bt/spider.js @@ -50,6 +50,13 @@ class Spider extends Emiter { this.walkInterval = config.spider.walkInterval; this.cpuLimit = config.spider.cpuLimit; this.cpuInterval = config.spider.cpuInterval; + this.foundSpeed = 0; + this.foundCounter = 0; + setInterval(() => { + this.foundSpeed = this.foundCounter; + this.foundCounter = 0; + console.log('found speed', this.foundSpeed) + }, 1000) this.announceHashes = [] this.searchHashes = [] @@ -114,16 +121,11 @@ class Spider extends Emiter { return if(!this.client || this.client.isIdle()) { - if( - !this.ignore - && (this.cpuLimit <= 0 || cpuUsage() < this.cpuLimit + this.cpuInterval) - && (config.trafficMax <= 0 || this.trafficSpeed == 0 || this.trafficSpeed < config.trafficMax) - ) + if(!this.ignore) { const node = this.table.shift() - //console.log(parseInt(Math.random() * this.table.caption / (this.table.nodes.length || 0))) //if (node) { - if (node && parseInt(Math.random() * this.table.nodes.length / 8) === 0) { + if (node && parseInt(Math.random() * this.table.nodes.length / 100) === 0) { //console.log('walk', this.table.nodes.length) this.findNode(Node.neighbor(node.id, this.table.id), {address: node.address, port: node.port}) } @@ -139,7 +141,6 @@ class Spider extends Emiter { this.table.add(node) } }) - //console.log('nodes', this.table.nodes.length) this.emit('nodes', nodes) // announce torrents @@ -153,7 +154,6 @@ class Spider extends Emiter { } onFoundPeers(peers, token, address) { - console.log('responce') if(token) { for(const hash of this.announceHashes) @@ -171,11 +171,8 @@ class Spider extends Emiter { } onFindNodeRequest(message, address) { - if(this.cpuLimit > 0 && cpuUsage() > this.cpuLimit) { - return - } - - if(config.trafficIgnoreDHT && config.trafficMax > 0 && this.trafficSpeed > 0 && this.trafficSpeed > config.trafficMax) { + if(this.foundSpeed > 500) + { return } @@ -202,11 +199,8 @@ class Spider extends Emiter { } onGetPeersRequest(message, address) { - if(this.cpuLimit > 0 && cpuUsage() > this.cpuLimit) { - return - } - - if(config.trafficIgnoreDHT && config.trafficMax > 0 && this.trafficSpeed > 0 && this.trafficSpeed > config.trafficMax) { + if(this.foundSpeed > 500) + { return } @@ -254,17 +248,14 @@ class Spider extends Emiter { if(this.client && !this.ignore) { cpuDebug('cpu usage:' + cpuUsage()) if(this.cpuLimit <= 0 || cpuUsage() <= this.cpuLimit + this.cpuInterval) { - // this.client.add(addressPair, infohash); + this.client.add(addressPair, infohash); } } } onPingRequest(message, address) { - if(this.cpuLimit > 0 && cpuUsage() > this.cpuLimit) { - return - } - - if(config.trafficIgnoreDHT && config.trafficMax > 0 && this.trafficSpeed > 0 && this.trafficSpeed > config.trafficMax) { + if(this.foundSpeed > 500) + { return } @@ -276,11 +267,13 @@ class Spider extends Emiter { const message = bencode.decode(data) if (message.y.toString() == 'r') { if(message.r.nodes) { + this.foundCounter++; this.onFoundNodes(message.r.nodes, message.r.token, address) } else if(message.r.values) { this.onFoundPeers(message.r.values, message.r.token, address) } } else if (message.y.toString() == 'q') { + this.foundCounter++; switch(message.q.toString()) { case 'get_peers': this.onGetPeersRequest(message, address) @@ -321,41 +314,6 @@ class Spider extends Emiter { }, 3000) this.join() this.walk() - - if(config.trafficMax > 0) - { - trafficDebug('inore dht traffic', config.trafficIgnoreDHT) - const path = `/sys/class/net/${config.trafficInterface}/statistics/rx_bytes` - if(fs.existsSync(path)) - { - trafficDebug('limitation', config.trafficMax / 1024, 'kbps/s') - let traffic = 0 - this.trafficInterval = setInterval(() => { - fs.readFile(path, (err, newTraffic) => { - if(err) - return - - if(traffic === 0) - traffic = newTraffic - - this.trafficSpeed = (newTraffic - traffic) / config.trafficUpdateTime - - trafficDebug('traffic rx', this.trafficSpeed / 1024, 'kbps/s') - - traffic = newTraffic - }) - }, 1000 * config.trafficUpdateTime) - } - } - - /* - this.announceSearchInterval = setInterval(() => { - for(const hash of this.announceHashes) - { - this.getPeersRequest(hash) - } - }, 3000) - */ } close(callback) @@ -366,10 +324,6 @@ class Spider extends Emiter { return } clearInterval(this.joinInterval) - if(this.trafficInterval) - clearInterval(this.trafficInterval) - //if(this.announceSearchInterval) - // clearInterval(this.announceSearchInterval) this.closing = true this.udp.close(() => { this.initialized = false From 9a95903cd5e298456e388e981333d084e75a778c Mon Sep 17 00:00:00 2001 From: Alexey Kasyanchuk Date: Thu, 1 Mar 2018 04:22:21 +0300 Subject: [PATCH 4/6] feat(config): some network limitations --- src/background/bt/spider.js | 25 +++++-------------------- src/background/config.js | 9 ++------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/src/background/bt/spider.js b/src/background/bt/spider.js index 6c51104..9cf05ad 100644 --- a/src/background/bt/spider.js +++ b/src/background/bt/spider.js @@ -5,14 +5,9 @@ const Emiter = require('events') const bencode = require('bencode') const {Table, Node} = require('./table') const Token = require('./token') -const cpuUsage = require('./cpu-usage') const config = require('../config') const fs = require('fs') -const _debug = require('debug') -const cpuDebug = _debug('spider:cpu') -const trafficDebug = _debug('spider:traffic') - const bootstraps = [{ address: 'router.bittorrent.com', port: 6881 @@ -45,21 +40,16 @@ class Spider extends Emiter { this.client = client this.ignore = false; // ignore all requests this.initialized = false; - this.trafficSpeed = 0 this.walkInterval = config.spider.walkInterval; - this.cpuLimit = config.spider.cpuLimit; - this.cpuInterval = config.spider.cpuInterval; this.foundSpeed = 0; this.foundCounter = 0; setInterval(() => { this.foundSpeed = this.foundCounter; this.foundCounter = 0; - console.log('found speed', this.foundSpeed) }, 1000) this.announceHashes = [] - this.searchHashes = [] } send(message, address) { @@ -124,9 +114,7 @@ class Spider extends Emiter { if(!this.ignore) { const node = this.table.shift() - //if (node) { - if (node && parseInt(Math.random() * this.table.nodes.length / 100) === 0) { - //console.log('walk', this.table.nodes.length) + if (node && (config.spider.nodesUsage === 0 || parseInt(Math.random() * this.table.nodes.length / config.spider.nodesUsage) === 0)) { this.findNode(Node.neighbor(node.id, this.table.id), {address: node.address, port: node.port}) } } @@ -171,7 +159,7 @@ class Spider extends Emiter { } onFindNodeRequest(message, address) { - if(this.foundSpeed > 500) + if(config.spider.packagesLimit !== 0 && this.foundSpeed > config.spider.packagesLimit) { return } @@ -199,7 +187,7 @@ class Spider extends Emiter { } onGetPeersRequest(message, address) { - if(this.foundSpeed > 500) + if(config.spider.packagesLimit !== 0 && this.foundSpeed > config.spider.packagesLimit) { return } @@ -246,15 +234,12 @@ class Spider extends Emiter { }; this.emit('ensureHash', infohash.toString('hex').toUpperCase(), addressPair) if(this.client && !this.ignore) { - cpuDebug('cpu usage:' + cpuUsage()) - if(this.cpuLimit <= 0 || cpuUsage() <= this.cpuLimit + this.cpuInterval) { - this.client.add(addressPair, infohash); - } + this.client.add(addressPair, infohash); } } onPingRequest(message, address) { - if(this.foundSpeed > 500) + if(config.spider.packagesLimit !== 0 && this.foundSpeed > config.spider.packagesLimit) { return } diff --git a/src/background/config.js b/src/background/config.js index 3cbb871..6eeb9fb 100644 --- a/src/background/config.js +++ b/src/background/config.js @@ -21,8 +21,8 @@ let config = { spider: { walkInterval: 5, - cpuLimit: 0, - cpuInterval: 10, + nodesUsage: 100, + packagesLimit: 500 }, downloader: { @@ -35,11 +35,6 @@ let config = { spaceQuota: false, spaceDiskLimit: 7 * 1024 * 1024 * 1024, - trafficInterface: 'enp2s0', - trafficMax: 0, - trafficUpdateTime: 3, //secs - trafficIgnoreDHT: false, - dbPath: '', client: { From df969bf9b95819b895c99598bb4dd1910f3573a5 Mon Sep 17 00:00:00 2001 From: Alexey Kasyanchuk Date: Thu, 1 Mar 2018 05:19:58 +0300 Subject: [PATCH 5/6] feat(config): new scanning options --- src/app/admin-page.js | 76 ++++++++++++++++++++++++++++++---------- src/background/config.js | 13 ++++++- 2 files changed, 70 insertions(+), 19 deletions(-) diff --git a/src/app/admin-page.js b/src/app/admin-page.js index 7ea44cb..9ae2518 100644 --- a/src/app/admin-page.js +++ b/src/app/admin-page.js @@ -134,24 +134,6 @@ export default class AdminPage extends Page { }} /> -
-
CPU usage limitation
- { - if(value === 100) - value = 0 - - this.options.spider.cpuLimit = value - this.forceUpdate() - }} - /> -
-
Download torrents directory
+
Torrent network scanner settings:
+
+
+
Scanner walk speed (current: {this.options.spider && this.options.spider.walkInterval}) [affected after program reload]
+ { + this.options.spider.walkInterval = value + this.forceUpdate() + }} + /> +
+
* Low value - fast initial scanning and high cpu usage. High Value - low cpu usage but very slow scanning. + Good value between 3-60. Defaul value: 5
+
+
+
+
Nodes usage (current: {this.options.spider && this.options.spider.nodesUsage})
+ { + this.options.spider.nodesUsage = value + this.forceUpdate() + }} + /> +
+
* Low Value - very low usage of nodes, low network traffic, slow torrent scanning. High value - high traffic, fast scanning, high routers usage. + Recomended value between 10-1000. Defaul value: 100. 0 - Ignore this option (no limit). +
+
+
+
+
Reduce network packages (current: {this.options.spider && this.options.spider.packagesLimit})
+ { + this.options.spider.packagesLimit = value + this.forceUpdate() + }} + /> +
+
* Low Value - ignore more usless network packages, lower traffic and routers usage. High Value - high traffic and router usage in prospectю + Recomended value between 300-2000. Defaul value: 500. 0 - Ignore this option (no limit). +
+
+ { this.settingsSavedMessage && diff --git a/src/background/config.js b/src/background/config.js index 6eeb9fb..cca922c 100644 --- a/src/background/config.js +++ b/src/background/config.js @@ -76,7 +76,18 @@ config.load = () => { const obj = JSON.parse(data); for(let prop in obj) { - config[prop] = obj[prop] + // объединяем объекты + if(typeof config[prop] === 'object' && typeof obj[prop] === 'object') + { + for(const subProp in obj[prop]) + { + config[prop][subProp] = obj[prop][subProp] + } + } + else + { + config[prop] = obj[prop] + } debug('rats.json:', prop, '=', obj[prop]) } } From 1d9cb84a185de05cde941def5c5e3e667969496b Mon Sep 17 00:00:00 2001 From: Alexey Kasyanchuk Date: Thu, 1 Mar 2018 05:24:31 +0300 Subject: [PATCH 6/6] cleanup --- src/background/bt/spider.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/background/bt/spider.js b/src/background/bt/spider.js index 9cf05ad..8558dfc 100644 --- a/src/background/bt/spider.js +++ b/src/background/bt/spider.js @@ -155,7 +155,6 @@ class Spider extends Emiter { const ips = Node.decodeCompactIP(peers) this.emit('peer', ips) - console.log('p', ips) } onFindNodeRequest(message, address) {