From 2d1a95594f2407407b6091bb22f4e8b445a7bb58 Mon Sep 17 00:00:00 2001 From: zicla Date: Sat, 1 Dec 2018 21:45:58 +0800 Subject: [PATCH] Finish the dashboard summary things. --- rest/context.go | 15 +++-- rest/dashboard_controller.go | 2 +- rest/dashboard_dao.go | 82 +++++++++++++------------- rest/dashboard_model.go | 4 +- rest/dashboard_service.go | 108 ++++++++++++++++++++++++++++++++++- rest/footprint_dao.go | 28 +++++++++ rest/footprint_model.go | 1 - rest/footprint_service.go | 1 - rest/image_cache_dao.go | 10 ++++ rest/logger.go | 10 ++-- rest/matter_dao.go | 18 ++++++ rest/util_time.go | 11 ++++ 12 files changed, 229 insertions(+), 61 deletions(-) diff --git a/rest/context.go b/rest/context.go index f51bb16..6fde9d2 100644 --- a/rest/context.go +++ b/rest/context.go @@ -46,17 +46,17 @@ func (this *Context) Init() { this.Router = NewRouter() } - func (this *Context) OpenDb() { var err error = nil this.DB, err = gorm.Open("mysql", CONFIG.MysqlUrl) - //是否打开sql日志 - this.DB.LogMode(false) if err != nil { - panic("failed to connect mysql database") + LOGGER.Panic("failed to connect mysql database") } + + //是否打开sql日志(在调试阶段可以打开，以方便查看执行的SQL) + this.DB.LogMode(false) } func (this *Context) CloseDb() { @@ -91,8 +91,7 @@ func (this *Context) registerBean(bean IBean) { } } else { - err := fmt.Sprintf("注册的【%s】不是Bean类型。", typeName) - panic(err) + LOGGER.Panic("注册的【%s】不是Bean类型。", typeName) } } @@ -154,8 +153,8 @@ func (this *Context) GetBean(bean IBean) IBean { if val, ok := this.BeanMap[typeName]; ok { return val } else { - err := fmt.Sprintf("【%s】没有注册。", typeName) - panic(err) + LOGGER.Panic("【%s】没有注册。", typeName) + return nil } } diff --git a/rest/dashboard_controller.go b/rest/dashboard_controller.go index 956f1af..8a9d6b3 100644 --- a/rest/dashboard_controller.go +++ b/rest/dashboard_controller.go @@ -41,6 +41,6 @@ func (this *DashboardController) RegisterRoutes() map[string]func(writer http.Re //过去七天分时调用量 func (this *DashboardController) InvokeList(writer http.ResponseWriter, request *http.Request) *WebResult { - return this.Success(this.dashboardDao.InvokeList()) + return this.Success("") } diff --git a/rest/dashboard_dao.go b/rest/dashboard_dao.go index 68d3298..0abe669 100644 --- a/rest/dashboard_dao.go +++ b/rest/dashboard_dao.go @@ -2,6 +2,7 @@ package rest import ( _ "github.com/jinzhu/gorm/dialects/mysql" + "github.com/nu7hatch/gouuid" "time" ) @@ -9,47 +10,46 @@ type DashboardDao struct { BaseDao } -//过去七天调用量 -func (this *DashboardDao) InvokeList() []*DashboardInvoke { +//创建 +func (this *DashboardDao) Create(dashboard *Dashboard) *Dashboard { - //过去几天 - var dayNum = 15; - var tableName = Footprint{}.TableName() - now := time.Now() - startDate := now.AddDate(0, 0, 1-dayNum) - rows, err := CONTEXT.DB.Raw("SELECT COUNT(uuid) AS invoke_num,COUNT(DISTINCT(ip)) AS uv,dt FROM "+tableName+" WHERE dt>= ? AND dt <= ? GROUP BY dt", - ConvertTimeToDateString(startDate), - ConvertTimeToDateString(now)).Rows() - this.PanicError(err) - defer rows.Close() + timeUUID, _ := uuid.NewV4() + dashboard.Uuid = string(timeUUID.String()) + dashboard.CreateTime = time.Now() + dashboard.UpdateTime = time.Now() + db := CONTEXT.DB.Create(dashboard) + this.PanicError(db.Error) - var invokeMap = make(map[string]*DashboardInvoke) - var dashboardInvokes []*DashboardInvoke - for rows.Next() { - var invokeNum int64 = 0; - var uv int64 = 0; - var dt string; - rows.Scan(&invokeNum, &uv, &dt) - invokeMap[dt] = &DashboardInvoke{ - InvokeNum: invokeNum, - Uv: uv, - Dt: dt, - } - } - for i := 1 - dayNum; i <= 0; i++ { - date := now.AddDate(0, 0, i) - dt := ConvertTimeToDateString(date) - v, ok := invokeMap[dt] - if ok { - dashboardInvokes = append(dashboardInvokes, v) - } else { - dashboardInvokes = append(dashboardInvokes, &DashboardInvoke{ - InvokeNum: 0, - Uv: 0, - Dt: dt, - }) - } - } - - return dashboardInvokes + return dashboard +} + +//修改一条记录 +func (this *DashboardDao) Save(dashboard *Dashboard) *Dashboard { + + dashboard.UpdateTime = time.Now() + db := CONTEXT.DB.Save(dashboard) + this.PanicError(db.Error) + + return dashboard +} + + +//删除一条记录 +func (this *DashboardDao) Delete(dashboard *Dashboard) { + + db := CONTEXT.DB.Delete(&dashboard) + this.PanicError(db.Error) +} + + +//按照dt查询 +func (this *DashboardDao) FindByDt(dt string) *Dashboard { + + // Read + var dashboard Dashboard + db := CONTEXT.DB.Where(&Dashboard{Dt: dt}).First(&dashboard) + if db.Error != nil { + return nil + } + return &dashboard } diff --git a/rest/dashboard_model.go b/rest/dashboard_model.go index 6a2e645..31b1e45 100644 --- a/rest/dashboard_model.go +++ b/rest/dashboard_model.go @@ -5,8 +5,8 @@ package rest */ type Dashboard struct { Base - VisitNum int64 `json:"visitNum"` - TotalVisitNum int64 `json:"totalVisitNum"` + InvokeNum int64 `json:"invokeNum"` + TotalInvokeNum int64 `json:"totalInvokeNum"` Uv int64 `json:"uv"` TotalUv int64 `json:"totalUv"` MatterNum int64 `json:"matterNum"` diff --git a/rest/dashboard_service.go b/rest/dashboard_service.go index c738935..5874d15 100644 --- a/rest/dashboard_service.go +++ b/rest/dashboard_service.go @@ -1,13 +1,21 @@ package rest +import ( + "time" +) + //@Service type DashboardService struct { Bean - dashboardDao *DashboardDao - userDao *UserDao + dashboardDao *DashboardDao + footprintDao *FootprintDao + matterDao *MatterDao + imageCacheDao *ImageCacheDao + userDao *UserDao + //每天凌晨定时整理器 + maintainTimer *time.Timer } - //初始化方法 func (this *DashboardService) Init() { this.Bean.Init() @@ -18,9 +26,103 @@ func (this *DashboardService) Init() { this.dashboardDao = b } + b = CONTEXT.GetBean(this.footprintDao) + if b, ok := b.(*FootprintDao); ok { + this.footprintDao = b + } + + b = CONTEXT.GetBean(this.matterDao) + if b, ok := b.(*MatterDao); ok { + this.matterDao = b + } + + b = CONTEXT.GetBean(this.imageCacheDao) + if b, ok := b.(*ImageCacheDao); ok { + this.imageCacheDao = b + } + b = CONTEXT.GetBean(this.userDao) if b, ok := b.(*UserDao); ok { this.userDao = b } + //立即执行数据清洗任务 + go this.maintain() +} + +//每日清洗离线数据表。 +func (this *DashboardService) maintain() { + + //准备好下次维护日志的时间。 + now := time.Now() + nextTime := FirstMinuteOfDay(Tomorrow()) + duration := nextTime.Sub(now) + this.logger.Info("每日数据汇总，下次时间：%s ", ConvertTimeToDateTimeString(nextTime)) + this.maintainTimer = time.AfterFunc(duration, func() { + go this.maintain() + }) + + //准备日期开始结尾 + startTime := FirstSecondOfDay(Yesterday()) + endTime := LastSecondOfDay(Yesterday()) + dt := ConvertTimeToDateString(startTime) + longTimeAgo := time.Now() + longTimeAgo = longTimeAgo.AddDate(-20, 0, 0) + + this.logger.Info("统计汇总表 %s -> %s", ConvertTimeToDateTimeString(startTime), ConvertTimeToDateTimeString(endTime)) + + //判断昨天的记录是否已经生成，如果生成了就直接删除掉 + dbDashboard := this.dashboardDao.FindByDt(dt) + if dbDashboard != nil { + this.logger.Info(" %s 的汇总已经存在了，删除以进行更新", dt) + this.dashboardDao.Delete(dbDashboard) + } + + invokeNum := this.footprintDao.CountBetweenTime(startTime, endTime) + this.logger.Info("调用数：%d", invokeNum) + + totalInvokeNum := this.footprintDao.CountBetweenTime(longTimeAgo, endTime) + this.logger.Info("历史总调用数：%d", totalInvokeNum) + + uv := this.footprintDao.UvBetweenTime(startTime, endTime) + this.logger.Info("UV：%d", uv) + + totalUv := this.footprintDao.UvBetweenTime(longTimeAgo, endTime) + this.logger.Info("历史总UV：%d", totalUv) + + matterNum := this.matterDao.CountBetweenTime(startTime, endTime) + this.logger.Info("文件数量数：%d", matterNum) + + totalMatterNum := this.matterDao.CountBetweenTime(longTimeAgo, endTime) + this.logger.Info("历史文件总数：%d", totalMatterNum) + + matterSize := this.matterDao.SizeBetweenTime(startTime, endTime) + this.logger.Info("文件大小：%d", matterSize) + + totalMatterSize := this.matterDao.SizeBetweenTime(longTimeAgo, endTime) + this.logger.Info("历史文件总大小：%d", totalMatterSize) + + cacheSize := this.imageCacheDao.SizeBetweenTime(startTime, endTime) + this.logger.Info("缓存大小：%d", cacheSize) + + totalCacheSize := this.imageCacheDao.SizeBetweenTime(longTimeAgo, endTime) + this.logger.Info("历史缓存总大小：%d", totalCacheSize) + + avgCost := this.footprintDao.AvgCostBetweenTime(startTime, endTime) + this.logger.Info("平均耗时：%d ms", avgCost) + + dashboard := &Dashboard{ + InvokeNum: invokeNum, + TotalInvokeNum: totalInvokeNum, + Uv: uv, + TotalUv: totalUv, + MatterNum: matterNum, + TotalMatterNum: totalMatterNum, + FileSize: matterSize + cacheSize, + TotalFileSize: totalMatterSize + totalCacheSize, + AvgCost: avgCost, + Dt: dt, + } + + this.dashboardDao.Create(dashboard) } diff --git a/rest/footprint_dao.go b/rest/footprint_dao.go index df267ac..169ee1b 100644 --- a/rest/footprint_dao.go +++ b/rest/footprint_dao.go @@ -88,3 +88,31 @@ func (this *FootprintDao) Delete(footprint *Footprint) { db := CONTEXT.DB.Delete(&footprint) this.PanicError(db.Error) } + +//获取一段时间中，总的数量 +func (this *FootprintDao) CountBetweenTime(startTime time.Time, endTime time.Time) int64 { + var count int64 + db := CONTEXT.DB.Model(&Footprint{}).Where("create_time >= ? AND create_time <= ?", startTime, endTime).Count(&count) + this.PanicError(db.Error) + return count +} + +//获取一段时间中UV的数量 +func (this *FootprintDao) UvBetweenTime(startTime time.Time, endTime time.Time) int64 { + var count int64 + db := CONTEXT.DB.Model(&Footprint{}).Where("create_time >= ? AND create_time <= ?", startTime, endTime).Select("COUNT(DISTINCT(ip))") + this.PanicError(db.Error) + row := db.Row() + row.Scan(&count) + return count +} + +//获取一段时间中平均耗时 +func (this *FootprintDao) AvgCostBetweenTime(startTime time.Time, endTime time.Time) int64 { + var cost float64 + db := CONTEXT.DB.Model(&Footprint{}).Where("create_time >= ? AND create_time <= ?", startTime, endTime).Select("AVG(cost)") + this.PanicError(db.Error) + row := db.Row() + row.Scan(&cost) + return int64(cost) +} diff --git a/rest/footprint_model.go b/rest/footprint_model.go index b857e6c..a9960d5 100644 --- a/rest/footprint_model.go +++ b/rest/footprint_model.go @@ -12,7 +12,6 @@ type Footprint struct { Params string `json:"params"` Cost int64 `json:"cost"` Success bool `json:"success"` - Dt string `json:"dt"` } // set File's table name to be `profiles` diff --git a/rest/footprint_service.go b/rest/footprint_service.go index 244ce2c..f406dd8 100644 --- a/rest/footprint_service.go +++ b/rest/footprint_service.go @@ -76,7 +76,6 @@ func (this *FootprintService) Trace(writer http.ResponseWriter, request *http.Re Params: paramsString, Cost: int64(duration / time.Millisecond), Success: success, - Dt: ConvertTimeToDateString(time.Now()), } footprint = this.footprintDao.Create(footprint) diff --git a/rest/image_cache_dao.go b/rest/image_cache_dao.go index 41e4781..ae54d15 100644 --- a/rest/image_cache_dao.go +++ b/rest/image_cache_dao.go @@ -192,3 +192,13 @@ func (this *ImageCacheDao) DeleteByMatterUuid(matterUuid string) { } } + +//获取一段时间中文件总大小 +func (this *ImageCacheDao) SizeBetweenTime(startTime time.Time, endTime time.Time) int64 { + var size int64 + db := CONTEXT.DB.Model(&ImageCache{}).Where("create_time >= ? AND create_time <= ?", startTime, endTime).Select("SUM(size)") + this.PanicError(db.Error) + row := db.Row() + row.Scan(&size) + return size +} diff --git a/rest/logger.go b/rest/logger.go index 03f2fb2..ce53917 100644 --- a/rest/logger.go +++ b/rest/logger.go @@ -32,10 +32,13 @@ func (this *Logger) log(prefix string, format string, v ...interface{}) { defer this.Unlock() //控制台中打印日志 - fmt.Printf(format+"\r\n", v...) + var consoleFormat = fmt.Sprintf("%s%s %s\r\n", prefix, ConvertTimeToTimeString(time.Now()), format) + fmt.Printf(consoleFormat, v...) this.goLogger.SetPrefix(prefix) - this.goLogger.Printf(format, v...) + //每一行我们加上换行符 + var fileFormat = fmt.Sprintf("%s\r\n", format) + this.goLogger.Printf(fileFormat, v...) } //处理日志的统一方法。 @@ -101,7 +104,7 @@ func (this *Logger) maintain() { now := time.Now() nextTime := FirstSecondOfDay(Tomorrow()) duration := nextTime.Sub(now) - go this.Info("%vs后将进行下一次日志维护 下次时间维护时间：%v ", int64(duration/time.Second), nextTime) + go this.Info("%vs 后将进行下一次日志维护 下次时间维护时间：%v ", int64(duration/time.Second), nextTime) this.maintainTimer = time.AfterFunc(duration, func() { go this.maintain() }) @@ -133,7 +136,6 @@ func (this *Logger) closeFile() { panic("尝试关闭日志时出错: " + err.Error()) } } - } func (this *Logger) Destroy() { diff --git a/rest/matter_dao.go b/rest/matter_dao.go index 93292b6..b9bc373 100644 --- a/rest/matter_dao.go +++ b/rest/matter_dao.go @@ -264,3 +264,21 @@ func (this *MatterDao) Delete(matter *Matter) { } } + +//获取一段时间中，总的数量 +func (this *MatterDao) CountBetweenTime(startTime time.Time, endTime time.Time) int64 { + var count int64 + db := CONTEXT.DB.Model(&Matter{}).Where("create_time >= ? AND create_time <= ?", startTime, endTime).Count(&count) + this.PanicError(db.Error) + return count +} + +//获取一段时间中文件总大小 +func (this *MatterDao) SizeBetweenTime(startTime time.Time, endTime time.Time) int64 { + var size int64 + db := CONTEXT.DB.Model(&Matter{}).Where("create_time >= ? AND create_time <= ?", startTime, endTime).Select("SUM(size)") + this.PanicError(db.Error) + row := db.Row() + row.Scan(&size) + return size +} \ No newline at end of file diff --git a/rest/util_time.go b/rest/util_time.go index 38772c8..3e815c0 100644 --- a/rest/util_time.go +++ b/rest/util_time.go @@ -20,6 +20,11 @@ func ConvertTimeToDateTimeString(time time.Time) string { return time.Local().Format("2006-01-02 15:04:05") } +//将一个时间字符串转换成日期时间对象(yyyy-MM-dd HH:mm:ss) +func ConvertTimeToTimeString(time time.Time) string { + return time.Local().Format("15:04:05") +} + //将一个时间字符串转换成日期对象(yyyy-MM-dd) func ConvertTimeToDateString(time time.Time) string { return time.Local().Format("2006-01-02") @@ -37,6 +42,12 @@ func FirstSecondOfDay(day time.Time) time.Time { return time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, local) } +//一天中的第一分钟 +func FirstMinuteOfDay(day time.Time) time.Time { + local, _ := time.LoadLocation("Local") + return time.Date(day.Year(), day.Month(), day.Day(), 0, 1, 0, 0, local) +} + //明天此刻的时间 func Tomorrow() time.Time { tomorrow := time.Now()