evan/rats-search
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(architecture): Big performance improvements over big databases and files highlight in search. (#63)

Browse Source 
Reduction database space over 50%. Change database version to v7.

BREAKING CHANGE: databases v6 and v7 are incompatible and need a lot of time for updating (may be even some days/a lot of hours on very big databases)
This commit is contained in:
Alexey Kasyanchuk
2018-11-11 23:44:10 +03:00
committed by GitHub
parent 13ca63b954
commit 6afe85798a
11 changed files with 271 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/background/spider.js
View File

@ -515,12 +515,13 @@ module.exports = function (send, recive, dataDirectory, version, env)
torrent.id = torrentsId++;
const recheckFiles = (callback) => {
sphinxSingle.query('SELECT count(*) as files_count FROM files WHERE hash = ?', [torrent.hash], function(err, rows) {
if(!rows)
sphinxSingle.query('SELECT id FROM files WHERE hash = ? limit 1', [torrent.hash], function(err, filesRecords) {
if(err) {
logTE('add', 'cannot check files in recheckFiles')
return
}
const db_files = rows[0]['files_count'];
if(db_files !== torrent.files)
if(!filesRecords || filesRecords.length == 0)
{
callback()
}
@ -534,12 +535,23 @@ module.exports = function (send, recive, dataDirectory, version, env)
return;
}
filesList.forEach((file) => {
file.id = filesId++;
file.pathIndex = file.path;
});
let path = '';
let size = '';
for(const file of filesList)
{
path += file.path + '\n';
size += file.size + '\n';
}
path = path.slice(0, -1);
size = size.slice(0, -1);
sphinxSingle.insertValues('files', filesList, function(err, result) {
sphinxSingle.insertValues('files', {
id: torrent.id,
hash: torrent.hash,
path,
pathIndex: path,
size
}, function(err, result) {
if(!result) {
console.error(err);
return