evan/tank
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add the task service for scan task.

Browse Source
This commit is contained in:
lishuang 2020-07-11 20:46:47 +08:00
parent 7fe673068b
commit 7f394b327b
5 changed files with 184 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
code/rest/matter_service.go
View File

@ -339,13 +339,13 @@ func (this *MatterService) Upload(request *http.Request, file io.Reader, user *U
}
}
matter := this.CreateNonDirMatter(dirMatter, filename, fileSize, privacy, user)
matter := this.createNonDirMatter(dirMatter, filename, fileSize, privacy, user)
return matter
}
// create a non dir matter.
func (this *MatterService) CreateNonDirMatter(dirMatter *Matter, filename string, fileSize int64, privacy bool, user *User) *Matter {
func (this *MatterService) createNonDirMatter(dirMatter *Matter, filename string, fileSize int64, privacy bool, user *User) *Matter {
dirRelativePath := dirMatter.Path
fileRelativePath := dirRelativePath + "/" + filename
@ -373,7 +373,7 @@ func (this *MatterService) CreateNonDirMatter(dirMatter *Matter, filename string
}
// create a non dir matter.
func (this *MatterService) UpdateNonDirMatter(matter *Matter, fileSize int64, user *User) *Matter {
func (this *MatterService) updateNonDirMatter(matter *Matter, fileSize int64, user *User) *Matter {
matter.Size = fileSize
@ -1163,8 +1163,8 @@ func (this *MatterService) scanPhysicsFolder(request *http.Request, dirInfo os.F
//only check the fileSize.
if !matter.Dir {
if matter.Size != fileInfo.Size() {
this.logger.Info("update matter: %s size %d -> %d", name, matter.Size, fileInfo.Size())
this.UpdateNonDirMatter(matter, fileInfo.Size(), user)
this.logger.Info("update matter: %s size:%d -> %d", name, matter.Size, fileInfo.Size())
this.updateNonDirMatter(matter, fileInfo.Size(), user)
}
}
@ -1181,8 +1181,8 @@ func (this *MatterService) scanPhysicsFolder(request *http.Request, dirInfo os.F
} else {
//not exist. add basic info.
this.logger.Info("Create matter: %s size %d", name, fileInfo.Size())
matter = this.CreateNonDirMatter(dirMatter, name, fileInfo.Size(), true, user)
this.logger.Info("Create matter: %s size:%d", name, fileInfo.Size())
matter = this.createNonDirMatter(dirMatter, name, fileInfo.Size(), true, user)
}

9
code/rest/preference_controller.go
View File

@ -15,6 +15,7 @@ type PreferenceController struct {
preferenceDao *PreferenceDao
matterDao *MatterDao
preferenceService *PreferenceService
taskService *TaskService
}
func (this *PreferenceController) Init() {
@ -34,6 +35,11 @@ func (this *PreferenceController) Init() {
this.preferenceService = b
}
b = core.CONTEXT.GetBean(this.taskService)
if b, ok := b.(*TaskService); ok {
this.taskService = b
}
}
func (this *PreferenceController) RegisterRoutes() map[string]func(writer http.ResponseWriter, request *http.Request) {
@ -175,6 +181,9 @@ func (this *PreferenceController) EditScanConfig(writer http.ResponseWriter, req
preference = this.preferenceService.Save(preference)
//reinit the scan task.
this.taskService.InitScanTask()
return this.Success(preference)
}

122
code/rest/task_service.go
View File

@ -2,7 +2,9 @@ package rest
import (
"github.com/eyebluecn/tank/code/core"
"github.com/eyebluecn/tank/code/tool/util"
"github.com/robfig/cron/v3"
"net/http"
)
// system tasks service
@ -11,6 +13,13 @@ type TaskService struct {
BaseBean
footprintService *FootprintService
dashboardService *DashboardService
preferenceService *PreferenceService
matterService *MatterService
userDao *UserDao
//whether scan task is running
scanTaskRunning bool
scanTaskCron *cron.Cron
}
func (this *TaskService) Init() {
@ -26,6 +35,21 @@ func (this *TaskService) Init() {
this.dashboardService = b
}
b = core.CONTEXT.GetBean(this.preferenceService)
if b, ok := b.(*PreferenceService); ok {
this.preferenceService = b
}
b = core.CONTEXT.GetBean(this.matterService)
if b, ok := b.(*MatterService); ok {
this.matterService = b
}
b = core.CONTEXT.GetBean(this.userDao)
if b, ok := b.(*UserDao); ok {
this.userDao = b
}
this.scanTaskRunning = false
}
//init the clean footprint task.
@ -53,16 +77,99 @@ func (this *TaskService) InitEtlTask() {
this.logger.Info("[cron job] Everyday 00:05 ETL dashboard data. entryId = %d", entryId)
}
//scan task.
func (this *TaskService) doScanTask() {
if this.scanTaskRunning {
this.logger.Info("scan task is processing. Give up this invoke.")
return
} else {
this.scanTaskRunning = true
}
defer func() {
if err := recover(); err != nil {
this.logger.Info("occur error when do scan task.")
}
this.logger.Info("finish do scan task.")
this.scanTaskRunning = false
}()
this.logger.Info("do the scan task.")
preference := this.preferenceService.Fetch()
scanConfig := preference.FetchScanConfig()
if !scanConfig.Enable {
this.logger.Info("scan task not enabled.")
return
}
//mock a request.
request := &http.Request{}
if scanConfig.Scope == SCAN_SCOPE_ALL {
//scan all user's root folder.
this.userDao.PageHandle("", "", func(user *User) {
core.RunWithRecovery(func() {
this.matterService.DeleteByPhysics(request, user)
this.matterService.ScanPhysics(request, user)
})
})
} else if scanConfig.Scope == SCAN_SCOPE_CUSTOM {
//scan custom user's folder.
for _, username := range scanConfig.Usernames {
user := this.userDao.FindByUsername(username)
if user == nil {
this.logger.Error("username = %s not exist.", username)
} else {
this.logger.Info("scan custom user folder. username = %s", username)
core.RunWithRecovery(func() {
this.matterService.DeleteByPhysics(request, user)
this.matterService.ScanPhysics(request, user)
})
}
}
}
}
//init the scan task.
func (this *TaskService) InitScanTask() {
expression := "15 0 * * *"
cronJob := cron.New()
entryId, err := cronJob.AddFunc(expression, this.dashboardService.Etl)
core.PanicError(err)
cronJob.Start()
if this.scanTaskCron != nil {
this.scanTaskCron.Stop()
this.scanTaskCron = nil
}
this.logger.Info("[cron job] Everyday 00:05 ETL dashboard data. entryId = %d", entryId)
preference := this.preferenceService.Fetch()
scanConfig := preference.FetchScanConfig()
if !scanConfig.Enable {
this.logger.Info("scan task not enabled.")
return
}
if !util.ValidateCron(scanConfig.Cron) {
this.logger.Info("cron spec %s error", scanConfig.Cron)
return
}
this.scanTaskCron = cron.New()
entryId, err := this.scanTaskCron.AddFunc(scanConfig.Cron, this.doScanTask)
core.PanicError(err)
this.scanTaskCron.Start()
this.logger.Info("[cron job] %s do scan task. entryId = %d", scanConfig.Cron, entryId)
}
func (this *TaskService) Bootstrap() {
@ -73,4 +180,7 @@ func (this *TaskService) Bootstrap() {
//load the etl task.
this.InitEtlTask()
//load the scan task.
this.InitScanTask()
}

36
code/rest/user_dao.go
View File

@ -5,6 +5,7 @@ import (
"github.com/eyebluecn/tank/code/tool/builder"
"github.com/eyebluecn/tank/code/tool/result"
"github.com/eyebluecn/tank/code/tool/uuid"
"math"
"time"
)
@ -74,6 +75,15 @@ func (this *UserDao) FindByUsername(username string) *User {
func (this *UserDao) Page(page int, pageSize int, username string, status string, sortArray []builder.OrderPair) *Pager {
count, users := this.PlainPage(page, pageSize, username, status, sortArray)
pager := NewPager(page, pageSize, count, users)
return pager
}
func (this *UserDao) PlainPage(page int, pageSize int, username string, status string, sortArray []builder.OrderPair) (int, []*User) {
var wp = &builder.WherePair{}
if username != "" {
@ -98,9 +108,31 @@ func (this *UserDao) Page(page int, pageSize int, username string, status string
this.PanicError(db.Error)
pager := NewPager(page, pageSize, count, users)
return count, users
}
return pager
//handle user page by page.
func (this *UserDao) PageHandle(username string, status string, fun func(user *User)) {
//delete share and bridges.
pageSize := 1000
sortArray := []builder.OrderPair{
{
Key: "uuid",
Value: DIRECTION_ASC,
},
}
count, _ := this.PlainPage(0, pageSize, username, status, sortArray)
if count > 0 {
var totalPages = int(math.Ceil(float64(count) / float64(pageSize)))
var page int
for page = 0; page < totalPages; page++ {
_, users := this.PlainPage(0, pageSize, username, status, sortArray)
for _, u := range users {
fun(u)
}
}
}
}
func (this *UserDao) CountByUsername(username string) int {

16
code/tool/util/util_cron.go Normal file
View File

@ -0,0 +1,16 @@
package util
import (
"github.com/robfig/cron/v3"
)
//validate a cron
func ValidateCron(spec string) bool {
_, err := cron.ParseStandard(spec)
if err != nil {
return false
}
return true
}