Merge branch 'opt'

src/app/admin-page.js

@@ -134,24 +134,6 @@ export default class AdminPage extends Page {
           }} />
         </div>
 
-        <div className='row inline w100p'>
-          <div style={{flex: 1}}>CPU usage limitation</div>
-          <Slider
-            min={0}
-            max={100}
-            step={1}
-            style={{width: 300}}
-            value={this.options.spider && this.options.spider.cpuLimit === 0 ? 100 : this.options.spider && this.options.spider.cpuLimit}
-            onChange={(event, value) => {
-              if(value === 100)
-                value = 0
-              
-              this.options.spider.cpuLimit = value
-              this.forceUpdate()
-            }}
-          />
-        </div>
-
         <div className='row inline w100p'>
           <div style={{flex: 1}}>Download torrents directory</div>
           <TextField
@@ -175,6 +157,64 @@ export default class AdminPage extends Page {
           }} />
         </div>
 
+        <div style={{marginTop: 10}}>Torrent network scanner settings:</div>
+        <div className='column w100p'>
+          <div className='row inline w100p'>
+            <div style={{flex: 1}}>Scanner walk speed (current: {this.options.spider && this.options.spider.walkInterval}) [affected after program reload]</div>
+            <Slider
+              min={1}
+              max={150}
+              step={1}
+              style={{width: 300}}
+              value={this.options.spider && this.options.spider.walkInterval}
+              onChange={(event, value) => {
+                this.options.spider.walkInterval = value
+                this.forceUpdate()
+              }}
+            />
+          </div>
+          <div className='fs0-75' style={{color: 'grey'}}>* Low value - fast initial scanning and high cpu usage. High Value - low cpu usage but very slow scanning. 
+            Good value between 3-60. Defaul value: 5</div>
+        </div>
+        <div className='column w100p'>
+          <div className='row inline w100p'>
+            <div style={{flex: 1}}>Nodes usage (current: {this.options.spider && this.options.spider.nodesUsage})</div>
+            <Slider
+              min={0}
+              max={1000}
+              step={1}
+              style={{width: 300}}
+              value={this.options.spider && this.options.spider.nodesUsage}
+              onChange={(event, value) => {
+                this.options.spider.nodesUsage = value
+                this.forceUpdate()
+              }}
+            />
+          </div>
+          <div className='fs0-75' style={{color: 'grey'}}>* Low Value - very low usage of nodes, low network traffic, slow torrent scanning. High value - high traffic, fast scanning, high routers usage. 
+          Recomended value between 10-1000. Defaul value: 100. 0 - Ignore this option (no limit).
+          </div>
+        </div>
+        <div className='column w100p'>
+          <div className='row inline w100p'>
+            <div style={{flex: 1}}>Reduce network packages (current: {this.options.spider && this.options.spider.packagesLimit})</div>
+            <Slider
+              min={0}
+              max={2000}
+              step={1}
+              style={{width: 300}}
+              value={this.options.spider && this.options.spider.packagesLimit}
+              onChange={(event, value) => {
+                this.options.spider.packagesLimit = value
+                this.forceUpdate()
+              }}
+            />
+          </div>
+          <div className='fs0-75' style={{color: 'grey'}}>* Low Value - ignore more usless network packages, lower traffic and routers usage. High Value - high traffic and router usage in prospectю
+          Recomended value between 300-2000. Defaul value: 500. 0 - Ignore this option (no limit).
+          </div>
+        </div>
+
         {
           this.settingsSavedMessage
           &&

src/background/bt/spider.js

@@ -5,14 +5,9 @@ const Emiter = require('events')
 const bencode = require('bencode')
 const {Table, Node} = require('./table')
 const Token = require('./token')
-const cpuUsage = require('./cpu-usage')
 const config = require('../config')
 const fs = require('fs')
 
-const _debug = require('debug')
-const cpuDebug = _debug('spider:cpu')
-const trafficDebug = _debug('spider:traffic')
-
 const bootstraps = [{
     address: 'router.bittorrent.com',
     port: 6881
@@ -45,11 +40,14 @@ class Spider extends Emiter {
         this.client = client
         this.ignore = false; // ignore all requests
         this.initialized = false;
-        this.trafficSpeed = 0
 
         this.walkInterval = config.spider.walkInterval;
-        this.cpuLimit = config.spider.cpuLimit;
+        this.foundSpeed = 0;
-        this.cpuInterval = config.spider.cpuInterval;
+        this.foundCounter = 0;
+        setInterval(() => { 
+            this.foundSpeed = this.foundCounter;
+            this.foundCounter = 0;
+        }, 1000)
 
         this.announceHashes = []
     }
@@ -72,7 +70,7 @@ class Spider extends Emiter {
         this.send(message, address)
     }
 
-    getPeersRequest(infoHash) {
+    getPeersRequest(infoHash, address) {
         const message = {
             t: generateTid(),
             y: 'q',
@@ -82,13 +80,7 @@ class Spider extends Emiter {
               info_hash: infoHash
             }
         }
-        for(const address of this.table.nodes)
+        this.send(message, address)
-        {
-            if(parseInt(Math.random() * 5) !== 1)
-                continue;
-
-            this.send(message, address)
-        }
     }
 
     announcePeer(infoHash, token, address, port)
@@ -119,14 +111,10 @@ class Spider extends Emiter {
             return
 
     	if(!this.client || this.client.isIdle()) {
-            if(
+            if(!this.ignore) 
-                !this.ignore 
-                && (this.cpuLimit <= 0 || cpuUsage() < this.cpuLimit + this.cpuInterval)
-                && (config.trafficMax <= 0 || this.trafficSpeed == 0 || this.trafficSpeed < config.trafficMax)
-            ) 
             {
                 const node = this.table.shift()
-                if (node) {
+                if (node && (config.spider.nodesUsage === 0 || parseInt(Math.random() * this.table.nodes.length / config.spider.nodesUsage) === 0)) {
                     this.findNode(Node.neighbor(node.id, this.table.id), {address: node.address, port: node.port})
                 }
             }
@@ -170,11 +158,8 @@ class Spider extends Emiter {
     }
 
     onFindNodeRequest(message, address) {
-        if(this.cpuLimit > 0 && cpuUsage() > this.cpuLimit) {
+        if(config.spider.packagesLimit !== 0 && this.foundSpeed > config.spider.packagesLimit)
-            return
+        {
-        }
-
-        if(config.trafficIgnoreDHT && config.trafficMax > 0 && this.trafficSpeed > 0 && this.trafficSpeed > config.trafficMax) {
             return
         }
 
@@ -192,14 +177,17 @@ class Spider extends Emiter {
                 nodes: Node.encodeNodes(this.table.first())
             }
         }, address)
+
+        // also check hashes of alive ones
+        for(const hash of this.announceHashes)
+        {
+            this.getPeersRequest(hash, address) 
+        }
     }
 
     onGetPeersRequest(message, address) {
-        if(this.cpuLimit > 0 && cpuUsage() > this.cpuLimit) {
+        if(config.spider.packagesLimit !== 0 && this.foundSpeed > config.spider.packagesLimit)
-            return
+        {
-        }
-
-        if(config.trafficIgnoreDHT && config.trafficMax > 0 && this.trafficSpeed > 0 && this.trafficSpeed > config.trafficMax) {
             return
         }
 
@@ -220,6 +208,12 @@ class Spider extends Emiter {
         }, address)
 
         this.emit('unensureHash', infohash.toString('hex').toUpperCase())
+
+        // also check hashes of alive ones
+        for(const hash of this.announceHashes)
+        {
+            this.getPeersRequest(hash, address) 
+        }
     }
 
     onAnnouncePeerRequest(message, address) {
@@ -239,19 +233,13 @@ class Spider extends Emiter {
         };
     	this.emit('ensureHash', infohash.toString('hex').toUpperCase(), addressPair)
         if(this.client && !this.ignore) {
-            cpuDebug('cpu usage:' + cpuUsage())
+            this.client.add(addressPair, infohash);
-            if(this.cpuLimit <= 0 || cpuUsage() <= this.cpuLimit + this.cpuInterval) {
-                this.client.add(addressPair, infohash);
-            }
         }
     }
 
     onPingRequest(message, address) {
-        if(this.cpuLimit > 0 && cpuUsage() > this.cpuLimit) {
+        if(config.spider.packagesLimit !== 0 && this.foundSpeed > config.spider.packagesLimit)
-            return
+        {
-        }
-
-        if(config.trafficIgnoreDHT && config.trafficMax > 0 && this.trafficSpeed > 0 && this.trafficSpeed > config.trafficMax) {
             return
         }
 
@@ -263,11 +251,13 @@ class Spider extends Emiter {
             const message = bencode.decode(data)
             if (message.y.toString() == 'r') {
                 if(message.r.nodes) {
+                    this.foundCounter++;
                     this.onFoundNodes(message.r.nodes, message.r.token, address)
                 } else if(message.r.values) {
                     this.onFoundPeers(message.r.values, message.r.token, address)
                 }
             } else if (message.y.toString() == 'q') {
+                this.foundCounter++;
             	switch(message.q.toString()) {
             		case 'get_peers':
             		this.onGetPeersRequest(message, address)
@@ -308,39 +298,6 @@ class Spider extends Emiter {
         }, 3000)
         this.join()
         this.walk()
-
-        if(config.trafficMax > 0)
-        {
-            trafficDebug('inore dht traffic', config.trafficIgnoreDHT)
-            const path = `/sys/class/net/${config.trafficInterface}/statistics/rx_bytes`
-            if(fs.existsSync(path))
-            {
-                trafficDebug('limitation', config.trafficMax / 1024, 'kbps/s')
-                let traffic = 0
-                this.trafficInterval = setInterval(() => { 
-                    fs.readFile(path, (err, newTraffic) => {
-                        if(err)
-                            return
-
-                        if(traffic === 0)
-                            traffic = newTraffic
-
-                        this.trafficSpeed = (newTraffic - traffic) / config.trafficUpdateTime
-
-                        trafficDebug('traffic rx', this.trafficSpeed / 1024, 'kbps/s')
-
-                        traffic = newTraffic
-                    })
-                }, 1000 * config.trafficUpdateTime)
-            }
-        }
-
-        this.announceSearchInterval = setInterval(() => { 
-            for(const hash of this.announceHashes)
-            {
-                this.getPeersRequest(hash) 
-            }
-        }, 3000)
     }
 
     close(callback)
@@ -351,10 +308,6 @@ class Spider extends Emiter {
             return
         }
         clearInterval(this.joinInterval)
-        if(this.trafficInterval)
-            clearInterval(this.trafficInterval)
-        if(this.announceSearchInterval)
-            clearInterval(this.announceSearchInterval)
         this.closing = true
         this.udp.close(() => {
             this.initialized = false

src/background/config.js

@@ -21,8 +21,8 @@ let config = {
 
 	spider: {
 		walkInterval: 5,
-		cpuLimit: 0,
+		nodesUsage: 100,
-		cpuInterval: 10,
+		packagesLimit: 500
 	},
 
 	downloader: {
@@ -35,11 +35,6 @@ let config = {
 	spaceQuota: false,
 	spaceDiskLimit: 7 * 1024 * 1024 * 1024,
 
-	trafficInterface: 'enp2s0',
-	trafficMax: 0,
-	trafficUpdateTime: 3, //secs
-	trafficIgnoreDHT: false,
-
 	dbPath: '',
 
 	client: {
@@ -81,7 +76,18 @@ config.load = () => {
 		const obj = JSON.parse(data);
 		for(let prop in obj) 
 		{
-			config[prop] = obj[prop]
+			// объединяем объекты
+			if(typeof config[prop] === 'object' && typeof obj[prop] === 'object')
+			{
+				for(const subProp in obj[prop])
+				{
+					config[prop][subProp] = obj[prop][subProp]
+				}
+			}
+			else
+			{
+				config[prop] = obj[prop]
+			}
 			debug('rats.json:', prop, '=', obj[prop])
 		}
 	}

src/background/p2p.js

@@ -0,0 +1,126 @@
+const config = require('./config');
+const net = require('net')
+const JsonSocket = require('json-socket')
+
+class p2p {
+	peers = []
+	ignoreAddresses = []
+	messageHandlers = {}
+	size = 0
+
+	constructor(send = () => {})
+	{
+		this.send = send
+		this.tcpServer = net.createServer();
+		this.tcpServer.on('connection', (socket) => {
+			//console.log('p2p server connection', socket.remoteAddress)
+			socket = new JsonSocket(socket);
+			socket.on('error', (err) => {})
+			socket.on('message', (message) => {    
+				if(message.type && this.messageHandlers[message.type])
+				{
+					this.messageHandlers[message.type](message.data, (data) => {
+						socket.sendMessage({
+							id: message.id,
+							data
+						});
+					})
+				}
+			});
+		})
+	}
+
+	listen() {
+		console.log('listen p2p on', config.spiderPort, 'port')
+		this.tcpServer.listen(config.spiderPort);
+	}
+
+	on(type, callback) {
+		this.messageHandlers[type] = callback
+	}
+
+	add(address) {
+		const { peers } = this
+
+		if(this.size > 10)
+			return;
+
+		if(address.port <= 1 || address.port > 65535)
+			return;
+
+		if(this.ignoreAddresses.includes(address.address))
+			return;
+
+		for(let peer of peers)
+		{
+			if(peer.address === address.address) {
+				peer.port = address.port;
+				return;
+			}
+		}
+		this.connect(address)
+	}
+
+	connect(address)
+	{
+		this.peers.push(address)
+		const socket = new JsonSocket(new net.Socket()); //Decorate a standard net.Socket with JsonSocket
+		socket.on('connect', () => { //Don't send until we're connected
+			// add to peers
+			this.size++;
+			this.send('peer', this.size)
+			console.log('new peer', address)
+
+			const callbacks = {}
+			socket.on('message', (message) => {
+				if(message.id && callbacks[message.id])
+				{
+					callbacks[message.id](message.data);
+					delete callbacks[message.id];
+				}
+			});
+			
+			const emit = (type, data, callback) => {
+				const id = Math.random().toString(36).substring(5)
+				if(callback)
+					callbacks[id] = callback;
+				socket.sendMessage({
+					id,
+					type,
+					data
+				});
+			}
+			address.emit = emit
+		});
+
+		socket.on('close', () => {
+			const index = this.peers.indexOf(address);
+			if(index >= 0)
+			{
+				if(this.peers[index].emit) // only autorized peers
+				{
+					this.size--;
+					this.send('peer', this.size)
+				}
+				this.peers.splice(index, 1);
+
+				console.log('close peer connection', address)
+			}
+		})
+		
+		socket.on('error', (err) => {})
+
+		socket.connect(address.port, address.address);
+	}
+
+	emit(type, data, callback)
+	{
+		for(const peer of this.peers)
+		{
+			if(peer.emit)
+				peer.emit(type, data, callback)
+		}
+	}
+}
+
+module.exports = p2p

src/background/spider.js

@@ -3,9 +3,8 @@ const client = new (require('./bt/client'))
 const spider = new (require('./bt/spider'))(client)
 const mysql = require('mysql');
 const getPeersStatisticUDP = require('./bt/udp-tracker-request')
-const net = require('net')
-const JsonSocket = require('json-socket')
 const crypto = require('crypto')
+const P2PServer = require('./p2p')
 const stun = require('stun')
 //var express = require('express');
 //var app = express();
@@ -253,30 +252,8 @@ setInterval(() => {
 	topCache = {};
 }, 24 * 60 * 60 * 1000);
 
-
+const p2p = new P2PServer(send)
-
+p2p.listen()
-// socket
-const messageHandlers = {}
-const onSocketMessage = (type, callback) => {
-	messageHandlers[type] = callback
-}
-const tcpServer = net.createServer();
-tcpServer.listen(config.spiderPort);
-tcpServer.on('connection', (socket) => {
-	socket = new JsonSocket(socket);
-	socket.on('error', (err) => {})
-	socket.on('message', (message) => {    
-		if(message.type && messageHandlers[message.type])
-		{
-			messageHandlers[message.type](message.data, (data) => {
-				socket.sendMessage({
-					id: message.id,
-					data
-				});
-			})
-		}
-	});
-})
 
 //io.on('connection', function(socket)
 //{
@@ -437,7 +414,7 @@ tcpServer.on('connection', (socket) => {
 		})
 	});
 
-	onSocketMessage('searchTorrent', ({text, navigation} = {}, callback) => {
+	p2p.on('searchTorrent', ({text, navigation} = {}, callback) => {
 		if(!text)
 			return;
 
@@ -541,7 +518,7 @@ tcpServer.on('connection', (socket) => {
 		})
 	});
 
-	onSocketMessage('searchFiles', ({text, navigation} = {}, callback) => {
+	p2p.on('searchFiles', ({text, navigation} = {}, callback) => {
 		if(!text)
 			return;
 
@@ -1046,86 +1023,6 @@ client.on('complete', function (metadata, infohash, rinfo) {
 });
 
 
-const p2p = {
-	peers: [],
-	ignoreAddresses: [],
-	add(address) {
-		const { peers } = this
-
-		if(peers.length > 10)
-			return;
-
-		if(address.port <= 1 || address.port > 65535)
-			return;
-
-		if(this.ignoreAddresses.includes(address.address))
-			return;
-
-		for(let peer of peers)
-		{
-			if(peer.address === address.address) {
-				peer.port = address.port;
-				return;
-			}
-		}
-		this.connect(address)
-	},
-	connect(address)
-	{
-		this.peers.push(address)
-		const socket = new JsonSocket(new net.Socket()); //Decorate a standard net.Socket with JsonSocket
-		socket.on('connect', () => { //Don't send until we're connected
-			// add to peers
-			send('peer', this.peers.length)
-			console.log('new peer', address)
-
-			const callbacks = {}
-			socket.on('message', (message) => {
-				if(message.id && callbacks[message.id])
-				{
-					callbacks[message.id](message.data);
-					delete callbacks[message.id];
-				}
-			});
-			
-			const emit = (type, data, callback) => {
-				const id = Math.random().toString(36).substring(5)
-				if(callback)
-					callbacks[id] = callback;
-				socket.sendMessage({
-					id,
-					type,
-					data
-				});
-			}
-			address.emit = emit
-		});
-
-		socket.on('close', () => {
-			const index = this.peers.indexOf(address);
-			if(index >= 0)
-			{
-				this.peers.splice(index, 1);
-
-				console.log('close peer connection', address)
-				send('peer', this.peers.length)
-			}
-		})
-		
-		socket.on('error', (err) => {})
-
-		socket.connect(address.port, address.address);
-	},
-	emit(type, data, callback)
-	{
-		for(const peer of this.peers)
-		{
-			if(peer.emit)
-				peer.emit(type, data, callback)
-		}
-	}
-}
-
 const { STUN_BINDING_REQUEST, STUN_ATTR_XOR_MAPPED_ADDRESS } = stun.constants
 const stunServer = stun.createServer()
 const stunRequest = stun.createMessage(STUN_BINDING_REQUEST)
@@ -1139,11 +1036,6 @@ stunServer.once('bindingResponse', stunMsg => {
 stunServer.send(stunRequest, 19302, 'stun.l.google.com')
 
 spider.on('peer', (IPs) => {
-	const { peers } = p2p;
-
-	if(peers.length > 10)
-		return
-
 	IPs.forEach(ip => p2p.add(ip))
 })