evan/rats-search
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(patch): fix memory issue

Browse Source
This commit is contained in:
Alexey Kasyanchuk 2018-07-07 16:43:56 +03:00
parent 805c55849a
commit 064c93c176
1 changed files with 41 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/background/dbPatcher.js
View File

@ -198,13 +198,32 @@ module.exports = async (callback, mainWindow, sphinxApp) => {
let torrentsArray = [] let torrentsArray = []
let patch = 1
await forBigTable(sphinx, 'torrents', async (torrent) => { await forBigTable(sphinx, 'torrents', async (torrent) => {
console.log('remember index', torrent.id, torrent.name, '[', i, 'of', torrents, ']') console.log('remember index', torrent.id, torrent.name, '[', i, 'of', torrents, ']')
if(patchWindow) if(patchWindow)
patchWindow.webContents.send('reindex', {field: torrent.name, index: i++, all: torrents, torrent: true}) patchWindow.webContents.send('reindex', {field: torrent.name, index: i++, all: torrents, torrent: true})
torrentsArray.push(torrent) torrentsArray.push(torrent)
// keep memory safe
if(torrentsArray.length >= 20000)
{
fs.writeFileSync(`${sphinxApp.directoryPath}/torrents.patch.${patch++}`, JSON.stringify(torrentsArray, null, 4), 'utf8');
console.log('write torrents dump', `${sphinxApp.directoryPath}/torrents.patch.${patch-1}`)
torrentsArray = []
}
}) })
// keep last elemets
if(torrentsArray.length > 0)
{
fs.writeFileSync(`${sphinxApp.directoryPath}/torrents.patch.${patch}`, JSON.stringify(torrentsArray, null, 4), 'utf8');
console.log('write torrents dump', `${sphinxApp.directoryPath}/torrents.patch.${patch}`)
torrentsArray = []
}
else
{
patch-- //no last patch
}
// stop sphinx // stop sphinx
await new Promise((resolve) => { await new Promise((resolve) => {
@ -237,14 +256,29 @@ module.exports = async (callback, mainWindow, sphinxApp) => {
console.log('sphinx restarted, patch db now') console.log('sphinx restarted, patch db now')
await asyncForEach(torrentsArray, async (torrent) => { for(let k = 1; k <= patch; k++)
console.log('update index', torrent.id, torrent.name, '[', i, 'of', torrents, ']') {
if(patchWindow) torrentsArray = JSON.parse(fs.readFileSync(`${sphinxApp.directoryPath}/torrents.patch.${k}`, 'utf8'))
patchWindow.webContents.send('reindex', {field: torrent.name, index: i++, all: torrents, torrent: true}) console.log('read torrents dump', `${sphinxApp.directoryPath}/torrents.patch.${k}`)
await asyncForEach(torrentsArray, async (torrent) => {
console.log('update index', torrent.id, torrent.name, '[', i, 'of', torrents, ']')
if(patchWindow)
patchWindow.webContents.send('reindex', {field: torrent.name, index: i++, all: torrents, torrent: true})
torrent.nameIndex = torrent.name torrent.nameIndex = torrent.name
await sphinx.query(`DELETE FROM torrents WHERE id = ${torrent.id}`) await sphinx.query(`DELETE FROM torrents WHERE id = ${torrent.id}`)
await sphinx.insertValues('torrents', torrent) await sphinx.insertValues('torrents', torrent)
})
}
await new Promise((resolve) => {
glob(`${sphinxApp.directoryPath}/torrents.patch.*`, function (er, files) {
files.forEach(file => {
console.log('clear dump file', file)
fs.unlinkSync(path.resolve(file))
})
resolve()
})
}) })
torrentsArray = null torrentsArray = null