evan/rats-search
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(db): converting db to version 8

Browse Source
This commit is contained in:
Alexey Kasyanchuk 2021-09-04 23:09:55 +03:00
parent f7166103df
commit 177f8d2f11
1 changed files with 43 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
src/background/dbPatcher.js
View File

@ -11,7 +11,7 @@ const {torrentTypeDetect, torrentTypeId, torrentIdToType, torrentCategoryId, tor
const startSphinx = require('./sphinx') const startSphinx = require('./sphinx')
const currentVersion = 7 const currentVersion = 8
module.exports = async (callback, mainWindow, sphinxApp) => { module.exports = async (callback, mainWindow, sphinxApp) => {
@ -120,7 +120,7 @@ module.exports = async (callback, mainWindow, sphinxApp) => {
const patch = async (version) => { const patch = async (version) => {
logT('patcher', 'db version', version) logT('patcher', 'db version', version)
const rebuildTorrentsFull = async () => { const rebuildFull = async (table = 'torrents', patchFunction = (torrent) => torrent.nameIndex = torrent.name) => {
if(sphinxApp.isExternal) if(sphinxApp.isExternal)
{ {
@ -129,30 +129,30 @@ module.exports = async (callback, mainWindow, sphinxApp) => {
} }
let i = 1 let i = 1
const torrents = (await sphinx.query("SELECT COUNT(*) AS c FROM torrents"))[0].c const torrents = (await sphinx.query(`SELECT COUNT(*) AS c FROM ${table}`))[0].c
let torrentsArray = [] let torrentsArray = []
let patch = 1 let patch = 1
await forBigTable(sphinx, 'torrents', async (torrent) => { await forBigTable(sphinx, table, async (torrent) => {
logT('patcher', 'remember index', torrent.id, torrent.name, '[', i, 'of', torrents, ']') logT('patcher', 'remember index', torrent.id, torrent.name || torrent.hash, '[', i, 'of', torrents, ']')
if(patchWindow) if(patchWindow)
patchWindow.webContents.send('reindex', {field: torrent.name, index: i++, all: torrents, torrent: true}) patchWindow.webContents.send('reindex', {field: torrent.name || torrent.hash, index: i++, all: torrents, torrent: true})
torrentsArray.push(torrent) torrentsArray.push(torrent)
// keep memory safe // keep memory safe
if(torrentsArray.length >= 20000) if(torrentsArray.length >= 20000)
{ {
fs.writeFileSync(`${sphinxApp.directoryPath}/torrents.patch.${patch++}`, JSON.stringify(torrentsArray, null, 4), 'utf8'); fs.writeFileSync(`${sphinxApp.directoryPath}/${table}.patch.${patch++}`, JSON.stringify(torrentsArray, null, 4), 'utf8');
logT('patcher', 'write torrents dump', `${sphinxApp.directoryPath}/torrents.patch.${patch - 1}`) logT('patcher', `write ${table} dump`, `${sphinxApp.directoryPath}/${table}.patch.${patch - 1}`)
torrentsArray = [] torrentsArray = []
} }
}) })
// keep last elemets // keep last elemets
if(torrentsArray.length > 0) if(torrentsArray.length > 0)
{ {
fs.writeFileSync(`${sphinxApp.directoryPath}/torrents.patch.${patch}`, JSON.stringify(torrentsArray, null, 4), 'utf8'); fs.writeFileSync(`${sphinxApp.directoryPath}/${table}.patch.${patch}`, JSON.stringify(torrentsArray, null, 4), 'utf8');
logT('patcher', 'write torrents dump', `${sphinxApp.directoryPath}/torrents.patch.${patch}`) logT('patcher', 'write torrents dump', `${sphinxApp.directoryPath}/${table}.patch.${patch}`)
torrentsArray = [] torrentsArray = []
} }
else else
@ -170,44 +170,45 @@ module.exports = async (callback, mainWindow, sphinxApp) => {
logT('patcher', 'sphinx stoped for patching') logT('patcher', 'sphinx stoped for patching')
await new Promise((resolve) => { await new Promise((resolve) => {
glob(`${sphinxApp.directoryPathDb}/torrents.*`, function (er, files) { glob(`${sphinxApp.directoryPathDb}/${table}.*`, function (er, files) {
files.forEach(file => { files.forEach(file => {
logT('patcher', 'clear torrents file', file) logT('patcher', `clear ${table} file`, file)
fs.unlinkSync(path.resolve(file)) fs.unlinkSync(path.resolve(file))
}) })
resolve() resolve()
}) })
}) })
logT('patcher', 'cleaned torrents db structure, rectreating again') logT('patcher', `cleaned ${table} db structure, rectreating again`)
i = 1 i = 1
await new Promise(async (resolve) => { await new Promise(async (resolve) => {
// reopen sphinx // reopen sphinx
sphinxApp = await sphinxApp.start(async () => { const spRun = await sphinxApp.start(async () => {
sphinx = await single().waitConnection() sphinx = await single().waitConnection()
resolve() resolve()
}) // same args }) // same args
sphinxApp = spRun.sphinx;
}) })
logT('patcher', 'sphinx restarted, patch db now') logT('patcher', 'sphinx restarted, patch db now')
for(let k = 1; k <= patch; k++) for(let k = 1; k <= patch; k++)
{ {
torrentsArray = JSON.parse(fs.readFileSync(`${sphinxApp.directoryPath}/torrents.patch.${k}`, 'utf8')) torrentsArray = JSON.parse(fs.readFileSync(`${sphinxApp.directoryPath}/${table}.patch.${k}`, 'utf8'))
logT('patcher', 'read torrents dump', `${sphinxApp.directoryPath}/torrents.patch.${k}`) logT('patcher', 'read torrents dump', `${sphinxApp.directoryPath}/${table}.patch.${k}`)
await asyncForEach(torrentsArray, async (torrent) => { await asyncForEach(torrentsArray, async (torrent) => {
logT('patcher', 'update index', torrent.id, torrent.name, '[', i, 'of', torrents, ']') logT('patcher', 'update index', torrent.id, torrent.name || torrent.hash, '[', i, 'of', torrents, ']')
if(patchWindow) if(patchWindow)
patchWindow.webContents.send('reindex', {field: torrent.name, index: i++, all: torrents, torrent: true}) patchWindow.webContents.send('reindex', {field: torrent.name, index: i++, all: torrents, torrent: true})
torrent.nameIndex = torrent.name patchFunction(torrent);
await sphinx.query(`DELETE FROM torrents WHERE id = ${torrent.id}`) //await sphinx.query(`DELETE FROM ${table} WHERE id = ${torrent.id}`)
await sphinx.insertValues('torrents', torrent) await sphinx.insertValues(table, torrent)
}) })
} }
await new Promise((resolve) => { await new Promise((resolve) => {
glob(`${sphinxApp.directoryPath}/torrents.patch.*`, function (er, files) { glob(`${sphinxApp.directoryPath}/${table}.patch.*`, function (er, files) {
files.forEach(file => { files.forEach(file => {
logT('patcher', 'clear dump file', file) logT('patcher', 'clear dump file', file)
fs.unlinkSync(path.resolve(file)) fs.unlinkSync(path.resolve(file))
@ -218,11 +219,11 @@ module.exports = async (callback, mainWindow, sphinxApp) => {
torrentsArray = null torrentsArray = null
logT('patcher', 'optimizing torrents') logT('patcher', `optimizing ${table}`)
if(patchWindow) if(patchWindow)
patchWindow.webContents.send('optimize', {field: 'torrents'}) patchWindow.webContents.send('optimize', {field: table})
sphinx.query(`OPTIMIZE INDEX torrents`) sphinx.query(`OPTIMIZE INDEX ${table}`)
await sphinxApp.waitOptimized('torrents') await sphinxApp.waitOptimized(table)
} }
switch(version) switch(version)
@ -251,6 +252,7 @@ module.exports = async (callback, mainWindow, sphinxApp) => {
if(patchWindow) if(patchWindow)
patchWindow.webContents.send('reindex', {field: file.path, index: i++, all: files}) patchWindow.webContents.send('reindex', {field: file.path, index: i++, all: files})
file.pathIndex = file.path
await sphinx.query(`DELETE FROM files WHERE id = ${file.id}`) await sphinx.query(`DELETE FROM files WHERE id = ${file.id}`)
await sphinx.insertValues('files', file) await sphinx.insertValues('files', file)
}) })
@ -295,7 +297,7 @@ module.exports = async (callback, mainWindow, sphinxApp) => {
delete torrent.contenttype delete torrent.contenttype
torrent.filesList = (await sphinx.query(`SELECT * FROM files WHERE hash = '${torrent.hash}'`)) || [] torrent.filesList = (await sphinx.query(`SELECT * FROM files WHERE hash = '${torrent.hash}'`)) || []
torrentTypeDetect(torrent, torrent.filesList) torrentTypeDetect(torrent, torrent.filesList)
if(torrentIdToType(torrent.contentType) == 'bad') if(torrent.contentType == 'bad')
{ {
logT('patcher', 'remove bad torrent', torrent.name) logT('patcher', 'remove bad torrent', torrent.name)
bad++ bad++
@ -312,13 +314,13 @@ module.exports = async (callback, mainWindow, sphinxApp) => {
case 4: case 4:
{ {
openPatchWindow() openPatchWindow()
await rebuildTorrentsFull() await rebuildFull()
await setVersion(5) await setVersion(5)
} }
case 5: case 5:
{ {
openPatchWindow() openPatchWindow()
await rebuildTorrentsFull() await rebuildFull()
await setVersion(6) await setVersion(6)
} }
case 6: case 6:
@ -388,6 +390,7 @@ module.exports = async (callback, mainWindow, sphinxApp) => {
id: newId++, id: newId++,
hash, hash,
path: filesMap[hash][0].path, path: filesMap[hash][0].path,
pathIndex: filesMap[hash][0].path,
size_new: filesMap[hash][0].size.toString() size_new: filesMap[hash][0].size.toString()
}); });
logT('patcher', 'patched file', fileIndex, 'from', count, 'hash', hash, 'cIndex', ++hashCount); logT('patcher', 'patched file', fileIndex, 'from', count, 'hash', hash, 'cIndex', ++hashCount);
@ -435,7 +438,7 @@ module.exports = async (callback, mainWindow, sphinxApp) => {
return return
file.size = file.size_new.toString(); file.size = file.size_new.toString();
delete file.size_new; delete file.size_new;
await sphinx.replaceValues('files', file, {particial: false}); await sphinx.replaceValues('files', file, {particial: false, sphinxIndex: {pathIndex: 'path'}});
if(patchWindow) if(patchWindow)
patchWindow.webContents.send('reindex', {field: file.id, index: fileIndex, all: count, longTime: false, canBreak: true}) patchWindow.webContents.send('reindex', {field: file.id, index: fileIndex, all: count, longTime: false, canBreak: true})
logT('patcher', 'restore patched file', fileIndex++, 'from', count, 'hash', file.hash); logT('patcher', 'restore patched file', fileIndex++, 'from', count, 'hash', file.hash);
@ -447,6 +450,17 @@ module.exports = async (callback, mainWindow, sphinxApp) => {
sphinx.query(`OPTIMIZE INDEX files`) sphinx.query(`OPTIMIZE INDEX files`)
await sphinxApp.waitOptimized('files') await sphinxApp.waitOptimized('files')
} }
case 7:
{
openPatchWindow()
await rebuildFull('torrents', (torrent) => {
torrent.nameIndex = torrent.name
torrent.contenttype = torrentTypeId(torrent.contenttype);
torrent.contentcategory = torrentCategoryId(torrent.contentcategory);
});
await rebuildFull('files', (file) => {});
await setVersion(8)
}
} }
logT('patcher', 'db patch done') logT('patcher', 'db patch done')
sphinx.destroy() sphinx.destroy()