Add the task service for scan task.

code/rest/matter_service.go

@@ -339,13 +339,13 @@ func (this *MatterService) Upload(request *http.Request, file io.Reader, user *U
 		}
 	}
 
-	matter := this.CreateNonDirMatter(dirMatter, filename, fileSize, privacy, user)
+	matter := this.createNonDirMatter(dirMatter, filename, fileSize, privacy, user)
 
 	return matter
 }
 
 // create a non dir matter.
-func (this *MatterService) CreateNonDirMatter(dirMatter *Matter, filename string, fileSize int64, privacy bool, user *User) *Matter {
+func (this *MatterService) createNonDirMatter(dirMatter *Matter, filename string, fileSize int64, privacy bool, user *User) *Matter {
 	dirRelativePath := dirMatter.Path
 	fileRelativePath := dirRelativePath + "/" + filename
 
@@ -373,7 +373,7 @@ func (this *MatterService) CreateNonDirMatter(dirMatter *Matter, filename string
 }
 
 // create a non dir matter.
-func (this *MatterService) UpdateNonDirMatter(matter *Matter, fileSize int64, user *User) *Matter {
+func (this *MatterService) updateNonDirMatter(matter *Matter, fileSize int64, user *User) *Matter {
 
 	matter.Size = fileSize
 
@@ -1163,8 +1163,8 @@ func (this *MatterService) scanPhysicsFolder(request *http.Request, dirInfo os.F
 			//only check the fileSize.
 			if !matter.Dir {
 				if matter.Size != fileInfo.Size() {
-					this.logger.Info("update matter: %s size %d -> %d", name, matter.Size, fileInfo.Size())
+					this.logger.Info("update matter: %s size:%d -> %d", name, matter.Size, fileInfo.Size())
-					this.UpdateNonDirMatter(matter, fileInfo.Size(), user)
+					this.updateNonDirMatter(matter, fileInfo.Size(), user)
 				}
 			}
 
@@ -1181,8 +1181,8 @@ func (this *MatterService) scanPhysicsFolder(request *http.Request, dirInfo os.F
 			} else {
 
 				//not exist. add basic info.
-				this.logger.Info("Create matter: %s size %d", name, fileInfo.Size())
+				this.logger.Info("Create matter: %s size:%d", name, fileInfo.Size())
-				matter = this.CreateNonDirMatter(dirMatter, name, fileInfo.Size(), true, user)
+				matter = this.createNonDirMatter(dirMatter, name, fileInfo.Size(), true, user)
 
 			}
 

code/rest/preference_controller.go

@@ -15,6 +15,7 @@ type PreferenceController struct {
 	preferenceDao     *PreferenceDao
 	matterDao         *MatterDao
 	preferenceService *PreferenceService
+	taskService       *TaskService
 }
 
 func (this *PreferenceController) Init() {
@@ -34,6 +35,11 @@ func (this *PreferenceController) Init() {
 		this.preferenceService = b
 	}
 
+	b = core.CONTEXT.GetBean(this.taskService)
+	if b, ok := b.(*TaskService); ok {
+		this.taskService = b
+	}
+
 }
 
 func (this *PreferenceController) RegisterRoutes() map[string]func(writer http.ResponseWriter, request *http.Request) {
@@ -175,6 +181,9 @@ func (this *PreferenceController) EditScanConfig(writer http.ResponseWriter, req
 
 	preference = this.preferenceService.Save(preference)
 
+	//reinit the scan task.
+	this.taskService.InitScanTask()
+
 	return this.Success(preference)
 }
 

code/rest/task_service.go

@@ -2,15 +2,24 @@ package rest
 
 import (
 	"github.com/eyebluecn/tank/code/core"
+	"github.com/eyebluecn/tank/code/tool/util"
 	"github.com/robfig/cron/v3"
+	"net/http"
 )
 
 // system tasks service
 //@Service
 type TaskService struct {
 	BaseBean
-	footprintService *FootprintService
+	footprintService  *FootprintService
-	dashboardService *DashboardService
+	dashboardService  *DashboardService
+	preferenceService *PreferenceService
+	matterService     *MatterService
+	userDao           *UserDao
+
+	//whether scan task is running
+	scanTaskRunning bool
+	scanTaskCron    *cron.Cron
 }
 
 func (this *TaskService) Init() {
@@ -26,6 +35,21 @@ func (this *TaskService) Init() {
 		this.dashboardService = b
 	}
 
+	b = core.CONTEXT.GetBean(this.preferenceService)
+	if b, ok := b.(*PreferenceService); ok {
+		this.preferenceService = b
+	}
+
+	b = core.CONTEXT.GetBean(this.matterService)
+	if b, ok := b.(*MatterService); ok {
+		this.matterService = b
+	}
+	b = core.CONTEXT.GetBean(this.userDao)
+	if b, ok := b.(*UserDao); ok {
+		this.userDao = b
+	}
+
+	this.scanTaskRunning = false
 }
 
 //init the clean footprint task.
@@ -53,16 +77,99 @@ func (this *TaskService) InitEtlTask() {
 	this.logger.Info("[cron job] Everyday 00:05 ETL dashboard data. entryId = %d", entryId)
 }
 
+//scan task.
+func (this *TaskService) doScanTask() {
+
+	if this.scanTaskRunning {
+		this.logger.Info("scan task is processing. Give up this invoke.")
+		return
+	} else {
+		this.scanTaskRunning = true
+	}
+
+	defer func() {
+		if err := recover(); err != nil {
+			this.logger.Info("occur error when do scan task.")
+		}
+		this.logger.Info("finish do scan task.")
+		this.scanTaskRunning = false
+	}()
+
+	this.logger.Info("do the scan task.")
+	preference := this.preferenceService.Fetch()
+	scanConfig := preference.FetchScanConfig()
+
+	if !scanConfig.Enable {
+		this.logger.Info("scan task not enabled.")
+		return
+	}
+
+	//mock a request.
+	request := &http.Request{}
+
+	if scanConfig.Scope == SCAN_SCOPE_ALL {
+		//scan all user's root folder.
+		this.userDao.PageHandle("", "", func(user *User) {
+
+			core.RunWithRecovery(func() {
+
+				this.matterService.DeleteByPhysics(request, user)
+				this.matterService.ScanPhysics(request, user)
+
+			})
+
+		})
+
+	} else if scanConfig.Scope == SCAN_SCOPE_CUSTOM {
+		//scan custom user's folder.
+
+		for _, username := range scanConfig.Usernames {
+			user := this.userDao.FindByUsername(username)
+			if user == nil {
+				this.logger.Error("username = %s not exist.", username)
+			} else {
+				this.logger.Info("scan custom user folder. username = %s", username)
+
+				core.RunWithRecovery(func() {
+
+					this.matterService.DeleteByPhysics(request, user)
+					this.matterService.ScanPhysics(request, user)
+
+				})
+
+			}
+		}
+	}
+
+}
+
 //init the scan task.
 func (this *TaskService) InitScanTask() {
 
-	expression := "15 0 * * *"
+	if this.scanTaskCron != nil {
-	cronJob := cron.New()
+		this.scanTaskCron.Stop()
-	entryId, err := cronJob.AddFunc(expression, this.dashboardService.Etl)
+		this.scanTaskCron = nil
-	core.PanicError(err)
+	}
-	cronJob.Start()
 
-	this.logger.Info("[cron job] Everyday 00:05 ETL dashboard data. entryId = %d", entryId)
+	preference := this.preferenceService.Fetch()
+	scanConfig := preference.FetchScanConfig()
+
+	if !scanConfig.Enable {
+		this.logger.Info("scan task not enabled.")
+		return
+	}
+
+	if !util.ValidateCron(scanConfig.Cron) {
+		this.logger.Info("cron spec %s error", scanConfig.Cron)
+		return
+	}
+
+	this.scanTaskCron = cron.New()
+	entryId, err := this.scanTaskCron.AddFunc(scanConfig.Cron, this.doScanTask)
+	core.PanicError(err)
+	this.scanTaskCron.Start()
+
+	this.logger.Info("[cron job] %s do scan task. entryId = %d", scanConfig.Cron, entryId)
 }
 
 func (this *TaskService) Bootstrap() {
@@ -73,4 +180,7 @@ func (this *TaskService) Bootstrap() {
 	//load the etl task.
 	this.InitEtlTask()
 
+	//load the scan task.
+	this.InitScanTask()
+
 }

code/rest/user_dao.go

@@ -5,6 +5,7 @@ import (
 	"github.com/eyebluecn/tank/code/tool/builder"
 	"github.com/eyebluecn/tank/code/tool/result"
 	"github.com/eyebluecn/tank/code/tool/uuid"
+	"math"
 	"time"
 )
 
@@ -74,6 +75,15 @@ func (this *UserDao) FindByUsername(username string) *User {
 
 func (this *UserDao) Page(page int, pageSize int, username string, status string, sortArray []builder.OrderPair) *Pager {
 
+	count, users := this.PlainPage(page, pageSize, username, status, sortArray)
+
+	pager := NewPager(page, pageSize, count, users)
+
+	return pager
+}
+
+func (this *UserDao) PlainPage(page int, pageSize int, username string, status string, sortArray []builder.OrderPair) (int, []*User) {
+
 	var wp = &builder.WherePair{}
 
 	if username != "" {
@@ -98,9 +108,31 @@ func (this *UserDao) Page(page int, pageSize int, username string, status string
 
 	this.PanicError(db.Error)
 
-	pager := NewPager(page, pageSize, count, users)
+	return count, users
+}
 
-	return pager
+//handle user page by page.
+func (this *UserDao) PageHandle(username string, status string, fun func(user *User)) {
+
+	//delete share and bridges.
+	pageSize := 1000
+	sortArray := []builder.OrderPair{
+		{
+			Key:   "uuid",
+			Value: DIRECTION_ASC,
+		},
+	}
+	count, _ := this.PlainPage(0, pageSize, username, status, sortArray)
+	if count > 0 {
+		var totalPages = int(math.Ceil(float64(count) / float64(pageSize)))
+		var page int
+		for page = 0; page < totalPages; page++ {
+			_, users := this.PlainPage(0, pageSize, username, status, sortArray)
+			for _, u := range users {
+				fun(u)
+			}
+		}
+	}
 }
 
 func (this *UserDao) CountByUsername(username string) int {

code/tool/util/util_cron.go

@@ -0,0 +1,16 @@
+package util
+
+import (
+	"github.com/robfig/cron/v3"
+)
+
+//validate a cron
+func ValidateCron(spec string) bool {
+
+	_, err := cron.ParseStandard(spec)
+	if err != nil {
+		return false
+	}
+
+	return true
+}