From 8adcdac6f4660d6702feca0091efe705b528d6c9 Mon Sep 17 00:00:00 2001 From: Alexey Kasyanchuk Date: Sat, 7 Jul 2018 05:56:38 +0300 Subject: [PATCH] feat(search): part words search feature --- package-lock.json | 2 +- package.json | 1 + src/background/asyncForEach.js | 5 +++ src/background/dbPatcher.js | 73 +++++++++++++++++++++++++++++++++- src/background/sphinx.js | 24 +++++++++-- 5 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 src/background/asyncForEach.js diff --git a/package-lock.json b/package-lock.json index e5c1377..fc985ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "rats-search", - "version": "0.21.1", + "version": "0.23.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index fa43602..10bad63 100644 --- a/package.json +++ b/package.json @@ -119,6 +119,7 @@ "electron-log": "^2.2.14", "electron-updater": "^2.21.10", "fs-jetpack": "^1.2.0", + "glob": "^7.1.2", "iconv-lite": "^0.4.19", "ipaddr.js": "^1.5.4", "json-socket": "^0.3.0", diff --git a/src/background/asyncForEach.js b/src/background/asyncForEach.js new file mode 100644 index 0000000..1f0a08e --- /dev/null +++ b/src/background/asyncForEach.js @@ -0,0 +1,5 @@ +module.exports = async function asyncForEach(array, callback) { + for (let index = 0; index < array.length; index++) { + await callback(array[index], index, array) + } +} \ No newline at end of file diff --git a/src/background/dbPatcher.js b/src/background/dbPatcher.js index 5009c56..9bdb6ab 100644 --- a/src/background/dbPatcher.js +++ b/src/background/dbPatcher.js @@ -4,16 +4,19 @@ const { BrowserWindow } = require("electron"); const url = require('url') const path = require('path') const fs = require('fs') +const glob = require("glob") +const asyncForEach = require('./asyncForEach') const {torrentTypeDetect} = require('../app/content'); const getTorrent = require('./gettorrent') +const startSphinx = require('./sphinx') -const currentVersion = 4 +const currentVersion = 5 module.exports = async (callback, mainWindow, sphinxApp) => { - const sphinx = await single().waitConnection() + let sphinx = await single().waitConnection() const setVersion = async (version) => { await sphinx.query(`delete from version where id = 1`) @@ -186,6 +189,72 @@ module.exports = async (callback, mainWindow, sphinxApp) => { await setVersion(4) } + case 4: + { + openPatchWindow() + + let i = 1 + const torrents = (await sphinx.query("SELECT COUNT(*) AS c FROM torrents"))[0].c + + const torrentsArray = [] + + await forBigTable(sphinx, 'torrents', async (torrent) => { + console.log('remember index', torrent.id, torrent.name, '[', i, 'of', torrents, ']') + if(patchWindow) + patchWindow.webContents.send('reindex', {field: torrent.name, index: i++, all: torrents, torrent: true}) + + torrentsArray.push(torrent) + }) + + // stop sphinx + await new Promise((resolve) => { + // reopen sphinx + sphinx.destroy() // destory connection + sphinxApp.stop(resolve, true) + }) + + console.log('sphinx stoped for patching') + + await new Promise((resolve) => { + glob(`${sphinxApp.directoryPathDb}/torrents.*`, function (er, files) { + files.forEach(file => { + console.log('clear torrents file', file) + fs.unlinkSync(path.resolve(file)) + }) + resolve() + }) + }) + + console.log('cleaned torrents db structure, rectreating again') + i = 1 + await new Promise((resolve) => { + // reopen sphinx + sphinxApp = sphinxApp.start(async () => { + sphinx = await single().waitConnection() + resolve() + }) // same args + }) + + console.log('sphinx restarted, patch db now') + + await asyncForEach(torrentsArray, async (torrent) => { + console.log('update index', torrent.id, torrent.name, '[', i, 'of', torrents, ']') + if(patchWindow) + patchWindow.webContents.send('reindex', {field: torrent.name, index: i++, all: torrents, torrent: true}) + + torrent.nameIndex = torrent.name + await sphinx.query(`DELETE FROM torrents WHERE id = ${torrent.id}`) + await sphinx.insertValues('torrents', torrent) + }) + + console.log('optimizing torrents') + if(patchWindow) + patchWindow.webContents.send('optimize', {field: 'torrents'}) + sphinx.query(`OPTIMIZE INDEX torrents`) + await sphinxApp.waitOptimized('torrents') + + await setVersion(5) + } } console.log('db patch done') sphinx.destroy() diff --git a/src/background/sphinx.js b/src/background/sphinx.js index 28b6b0b..953ff71 100644 --- a/src/background/sphinx.js +++ b/src/background/sphinx.js @@ -14,7 +14,10 @@ const writeSphinxConfig = (path, dbPath) => { index torrents { type = rt - path = ${dbPath}/database/torrents + path = ${dbPath}/database/torrents + + min_prefix_len = 3 + expand_keywords = 1 rt_attr_string = hash rt_attr_string = name @@ -70,7 +73,7 @@ const writeSphinxConfig = (path, dbPath) => { { type = rt path = ${dbPath}/database/feed - + rt_field = feedIndex rt_attr_json = data } @@ -138,6 +141,8 @@ const writeSphinxConfig = (path, dbPath) => { } module.exports = (callback, dataDirectory, onClose) => { + const start = (callback) => { + const sphinxPath = path.resolve(appPath('searchd')) console.log('Sphinx Path:', sphinxPath) @@ -159,6 +164,7 @@ module.exports = (callback, dataDirectory, onClose) => { } const sphinx = spawn(sphinxPath, options) // remeber initizalizing of db + sphinx.start = start sphinx.isInitDb = isInitDb sphinx.directoryPath = appConfig.dbPath sphinx.directoryPathDb = appConfig.dbPath + '/database' @@ -186,12 +192,18 @@ module.exports = (callback, dataDirectory, onClose) => { sphinx.on('close', (code, signal) => { console.log(`sphinx closed with code ${code} and signal ${signal}`) - if(onClose) + if(onClose && !sphinx.replaceOnClose) // sometime we don't want to call default callback onClose() + if(sphinx.onClose) + sphinx.onClose() }) - sphinx.stop = () => { + sphinx.stop = (onFinish, replaceFinish) => { console.log('sphinx closing...') + if(onFinish) + sphinx.onClose = onFinish + if(replaceFinish) + sphinx.replaceOnClose = true // sometime we don't want to call default callback exec(`"${sphinxPath}" --config "${config}" --stopwait`) } @@ -203,4 +215,8 @@ module.exports = (callback, dataDirectory, onClose) => { }) return sphinx + + } + + return start(callback) } \ No newline at end of file