diff --git a/code/rest/matter_service.go b/code/rest/matter_service.go index d0972b0..916fd67 100644 --- a/code/rest/matter_service.go +++ b/code/rest/matter_service.go @@ -339,13 +339,13 @@ func (this *MatterService) Upload(request *http.Request, file io.Reader, user *U } } - matter := this.CreateNonDirMatter(dirMatter, filename, fileSize, privacy, user) + matter := this.createNonDirMatter(dirMatter, filename, fileSize, privacy, user) return matter } // create a non dir matter. -func (this *MatterService) CreateNonDirMatter(dirMatter *Matter, filename string, fileSize int64, privacy bool, user *User) *Matter { +func (this *MatterService) createNonDirMatter(dirMatter *Matter, filename string, fileSize int64, privacy bool, user *User) *Matter { dirRelativePath := dirMatter.Path fileRelativePath := dirRelativePath + "/" + filename @@ -373,7 +373,7 @@ func (this *MatterService) CreateNonDirMatter(dirMatter *Matter, filename string } // create a non dir matter. -func (this *MatterService) UpdateNonDirMatter(matter *Matter, fileSize int64, user *User) *Matter { +func (this *MatterService) updateNonDirMatter(matter *Matter, fileSize int64, user *User) *Matter { matter.Size = fileSize @@ -1163,8 +1163,8 @@ func (this *MatterService) scanPhysicsFolder(request *http.Request, dirInfo os.F //only check the fileSize. if !matter.Dir { if matter.Size != fileInfo.Size() { - this.logger.Info("update matter: %s size %d -> %d", name, matter.Size, fileInfo.Size()) - this.UpdateNonDirMatter(matter, fileInfo.Size(), user) + this.logger.Info("update matter: %s size:%d -> %d", name, matter.Size, fileInfo.Size()) + this.updateNonDirMatter(matter, fileInfo.Size(), user) } } @@ -1181,8 +1181,8 @@ func (this *MatterService) scanPhysicsFolder(request *http.Request, dirInfo os.F } else { //not exist. add basic info. - this.logger.Info("Create matter: %s size %d", name, fileInfo.Size()) - matter = this.CreateNonDirMatter(dirMatter, name, fileInfo.Size(), true, user) + this.logger.Info("Create matter: %s size:%d", name, fileInfo.Size()) + matter = this.createNonDirMatter(dirMatter, name, fileInfo.Size(), true, user) } diff --git a/code/rest/preference_controller.go b/code/rest/preference_controller.go index e53a04e..e3cd9d1 100644 --- a/code/rest/preference_controller.go +++ b/code/rest/preference_controller.go @@ -15,6 +15,7 @@ type PreferenceController struct { preferenceDao *PreferenceDao matterDao *MatterDao preferenceService *PreferenceService + taskService *TaskService } func (this *PreferenceController) Init() { @@ -34,6 +35,11 @@ func (this *PreferenceController) Init() { this.preferenceService = b } + b = core.CONTEXT.GetBean(this.taskService) + if b, ok := b.(*TaskService); ok { + this.taskService = b + } + } func (this *PreferenceController) RegisterRoutes() map[string]func(writer http.ResponseWriter, request *http.Request) { @@ -175,6 +181,9 @@ func (this *PreferenceController) EditScanConfig(writer http.ResponseWriter, req preference = this.preferenceService.Save(preference) + //reinit the scan task. + this.taskService.InitScanTask() + return this.Success(preference) } diff --git a/code/rest/task_service.go b/code/rest/task_service.go index 2b87e5e..7989b9f 100644 --- a/code/rest/task_service.go +++ b/code/rest/task_service.go @@ -2,15 +2,24 @@ package rest import ( "github.com/eyebluecn/tank/code/core" + "github.com/eyebluecn/tank/code/tool/util" "github.com/robfig/cron/v3" + "net/http" ) // system tasks service //@Service type TaskService struct { BaseBean - footprintService *FootprintService - dashboardService *DashboardService + footprintService *FootprintService + dashboardService *DashboardService + preferenceService *PreferenceService + matterService *MatterService + userDao *UserDao + + //whether scan task is running + scanTaskRunning bool + scanTaskCron *cron.Cron } func (this *TaskService) Init() { @@ -26,6 +35,21 @@ func (this *TaskService) Init() { this.dashboardService = b } + b = core.CONTEXT.GetBean(this.preferenceService) + if b, ok := b.(*PreferenceService); ok { + this.preferenceService = b + } + + b = core.CONTEXT.GetBean(this.matterService) + if b, ok := b.(*MatterService); ok { + this.matterService = b + } + b = core.CONTEXT.GetBean(this.userDao) + if b, ok := b.(*UserDao); ok { + this.userDao = b + } + + this.scanTaskRunning = false } //init the clean footprint task. @@ -53,16 +77,99 @@ func (this *TaskService) InitEtlTask() { this.logger.Info("[cron job] Everyday 00:05 ETL dashboard data. entryId = %d", entryId) } +//scan task. +func (this *TaskService) doScanTask() { + + if this.scanTaskRunning { + this.logger.Info("scan task is processing. Give up this invoke.") + return + } else { + this.scanTaskRunning = true + } + + defer func() { + if err := recover(); err != nil { + this.logger.Info("occur error when do scan task.") + } + this.logger.Info("finish do scan task.") + this.scanTaskRunning = false + }() + + this.logger.Info("do the scan task.") + preference := this.preferenceService.Fetch() + scanConfig := preference.FetchScanConfig() + + if !scanConfig.Enable { + this.logger.Info("scan task not enabled.") + return + } + + //mock a request. + request := &http.Request{} + + if scanConfig.Scope == SCAN_SCOPE_ALL { + //scan all user's root folder. + this.userDao.PageHandle("", "", func(user *User) { + + core.RunWithRecovery(func() { + + this.matterService.DeleteByPhysics(request, user) + this.matterService.ScanPhysics(request, user) + + }) + + }) + + } else if scanConfig.Scope == SCAN_SCOPE_CUSTOM { + //scan custom user's folder. + + for _, username := range scanConfig.Usernames { + user := this.userDao.FindByUsername(username) + if user == nil { + this.logger.Error("username = %s not exist.", username) + } else { + this.logger.Info("scan custom user folder. username = %s", username) + + core.RunWithRecovery(func() { + + this.matterService.DeleteByPhysics(request, user) + this.matterService.ScanPhysics(request, user) + + }) + + } + } + } + +} + //init the scan task. func (this *TaskService) InitScanTask() { - expression := "15 0 * * *" - cronJob := cron.New() - entryId, err := cronJob.AddFunc(expression, this.dashboardService.Etl) - core.PanicError(err) - cronJob.Start() + if this.scanTaskCron != nil { + this.scanTaskCron.Stop() + this.scanTaskCron = nil + } - this.logger.Info("[cron job] Everyday 00:05 ETL dashboard data. entryId = %d", entryId) + preference := this.preferenceService.Fetch() + scanConfig := preference.FetchScanConfig() + + if !scanConfig.Enable { + this.logger.Info("scan task not enabled.") + return + } + + if !util.ValidateCron(scanConfig.Cron) { + this.logger.Info("cron spec %s error", scanConfig.Cron) + return + } + + this.scanTaskCron = cron.New() + entryId, err := this.scanTaskCron.AddFunc(scanConfig.Cron, this.doScanTask) + core.PanicError(err) + this.scanTaskCron.Start() + + this.logger.Info("[cron job] %s do scan task. entryId = %d", scanConfig.Cron, entryId) } func (this *TaskService) Bootstrap() { @@ -73,4 +180,7 @@ func (this *TaskService) Bootstrap() { //load the etl task. this.InitEtlTask() + //load the scan task. + this.InitScanTask() + } diff --git a/code/rest/user_dao.go b/code/rest/user_dao.go index a9ab5ea..c08d6c8 100644 --- a/code/rest/user_dao.go +++ b/code/rest/user_dao.go @@ -5,6 +5,7 @@ import ( "github.com/eyebluecn/tank/code/tool/builder" "github.com/eyebluecn/tank/code/tool/result" "github.com/eyebluecn/tank/code/tool/uuid" + "math" "time" ) @@ -74,6 +75,15 @@ func (this *UserDao) FindByUsername(username string) *User { func (this *UserDao) Page(page int, pageSize int, username string, status string, sortArray []builder.OrderPair) *Pager { + count, users := this.PlainPage(page, pageSize, username, status, sortArray) + + pager := NewPager(page, pageSize, count, users) + + return pager +} + +func (this *UserDao) PlainPage(page int, pageSize int, username string, status string, sortArray []builder.OrderPair) (int, []*User) { + var wp = &builder.WherePair{} if username != "" { @@ -98,9 +108,31 @@ func (this *UserDao) Page(page int, pageSize int, username string, status string this.PanicError(db.Error) - pager := NewPager(page, pageSize, count, users) + return count, users +} - return pager +//handle user page by page. +func (this *UserDao) PageHandle(username string, status string, fun func(user *User)) { + + //delete share and bridges. + pageSize := 1000 + sortArray := []builder.OrderPair{ + { + Key: "uuid", + Value: DIRECTION_ASC, + }, + } + count, _ := this.PlainPage(0, pageSize, username, status, sortArray) + if count > 0 { + var totalPages = int(math.Ceil(float64(count) / float64(pageSize))) + var page int + for page = 0; page < totalPages; page++ { + _, users := this.PlainPage(0, pageSize, username, status, sortArray) + for _, u := range users { + fun(u) + } + } + } } func (this *UserDao) CountByUsername(username string) int { diff --git a/code/tool/util/util_cron.go b/code/tool/util/util_cron.go new file mode 100644 index 0000000..45e03ce --- /dev/null +++ b/code/tool/util/util_cron.go @@ -0,0 +1,16 @@ +package util + +import ( + "github.com/robfig/cron/v3" +) + +//validate a cron +func ValidateCron(spec string) bool { + + _, err := cron.ParseStandard(spec) + if err != nil { + return false + } + + return true +}