evan/tank
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Finish the sqlite feature.

Browse Source
This commit is contained in:
lishuang
2022-03-18 22:17:32 +08:00
parent 179f34fad2
commit c6a5db7740
11 changed files with 331 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
code/core/config.go
View File

@ -20,12 +20,16 @@ const (
type Config interface {
Installed() bool
ServerPort() int
//get the db type
DbType() string
//get the mysql url. eg. tank:tank123@tcp(127.0.0.1:3306)/tank?charset=utf8&parseTime=True&loc=Local
MysqlUrl() string
//get the sqlite path
SqliteFolder() string
//files storage location.
MatterPath() string
//table name strategy
NamingStrategy() schema.NamingStrategy
//when installed by user. Write configs to tank.json
FinishInstall(mysqlPort int, mysqlHost string, mysqlSchema string, mysqlUsername string, mysqlPassword string, mysqlCharset string)
FinishInstall(dbType string, mysqlPort int, mysqlHost string, mysqlSchema string, mysqlUsername string, mysqlPassword string, mysqlCharset string)
}

20
code/rest/dashboard_model.go
View File

@ -10,16 +10,16 @@ type Dashboard struct {
Sort int64 `json:"sort" gorm:"type:bigint(20) not null"`
UpdateTime time.Time `json:"updateTime" gorm:"type:timestamp not null;default:CURRENT_TIMESTAMP"`
CreateTime time.Time `json:"createTime" gorm:"type:timestamp not null;default:'2018-01-01 00:00:00'"`
InvokeNum int64 `json:"invokeNum" gorm:"type:bigint(20) not null"` //api invoke num.
TotalInvokeNum int64 `json:"totalInvokeNum" gorm:"type:bigint(20) not null;default:0"` //total invoke num up to now.
Uv int64 `json:"uv" gorm:"type:bigint(20) not null;default:0"` //today's uv
TotalUv int64 `json:"totalUv" gorm:"type:bigint(20) not null;default:0"` //total uv
MatterNum int64 `json:"matterNum" gorm:"type:bigint(20) not null;default:0"` //file's num
TotalMatterNum int64 `json:"totalMatterNum" gorm:"type:bigint(20) not null;default:0"` //file's total number
FileSize int64 `json:"fileSize" gorm:"type:bigint(20) not null;default:0"` //today's file size
TotalFileSize int64 `json:"totalFileSize" gorm:"type:bigint(20) not null;default:0"` //total file's size
AvgCost int64 `json:"avgCost" gorm:"type:bigint(20) not null;default:0"` //api time cost in ms
Dt string `json:"dt" gorm:"type:varchar(45) not null;index:idx_dt"` //date
InvokeNum int64 `json:"invokeNum" gorm:"type:bigint(20) not null"` //api invoke num.
TotalInvokeNum int64 `json:"totalInvokeNum" gorm:"type:bigint(20) not null;default:0"` //total invoke num up to now.
Uv int64 `json:"uv" gorm:"type:bigint(20) not null;default:0"` //today's uv
TotalUv int64 `json:"totalUv" gorm:"type:bigint(20) not null;default:0"` //total uv
MatterNum int64 `json:"matterNum" gorm:"type:bigint(20) not null;default:0"` //file's num
TotalMatterNum int64 `json:"totalMatterNum" gorm:"type:bigint(20) not null;default:0"` //file's total number
FileSize int64 `json:"fileSize" gorm:"type:bigint(20) not null;default:0"` //today's file size
TotalFileSize int64 `json:"totalFileSize" gorm:"type:bigint(20) not null;default:0"` //total file's size
AvgCost int64 `json:"avgCost" gorm:"type:bigint(20) not null;default:0"` //api time cost in ms
Dt string `json:"dt" gorm:"type:varchar(45) not null;index:idx_dashboard_dt"` //date. index should unique globally.
}
/**

2
code/rest/download_token_model.go
View File

@ -10,7 +10,7 @@ type DownloadToken struct {
UpdateTime time.Time `json:"updateTime" gorm:"type:timestamp not null;default:CURRENT_TIMESTAMP"`
CreateTime time.Time `json:"createTime" gorm:"type:timestamp not null;default:'2018-01-01 00:00:00'"`
UserUuid string `json:"userUuid" gorm:"type:char(36) not null"`
MatterUuid string `json:"matterUuid" gorm:"type:char(36) not null;index:idx_mu"`
MatterUuid string `json:"matterUuid" gorm:"type:char(36) not null;index:idx_download_token_mu"` //index should unique globally.
ExpireTime time.Time `json:"expireTime" gorm:"type:timestamp not null;default:'2018-01-01 00:00:00'"`
Ip string `json:"ip" gorm:"type:varchar(128) not null"`
}

2
code/rest/image_cache_model.go
View File

@ -13,7 +13,7 @@ type ImageCache struct {
Name string `json:"name" gorm:"type:varchar(255) not null"`
UserUuid string `json:"userUuid" gorm:"type:char(36)"`
Username string `json:"username" gorm:"type:varchar(45) not null"`
MatterUuid string `json:"matterUuid" gorm:"type:char(36);index:idx_mu"`
MatterUuid string `json:"matterUuid" gorm:"type:char(36);index:idx_image_cache_mu"` //index should unique globally.
MatterName string `json:"matterName" gorm:"type:varchar(255) not null"`
Mode string `json:"mode" gorm:"type:varchar(512)"`
Md5 string `json:"md5" gorm:"type:varchar(45)"`

81
code/rest/install_controller.go
View File

@ -9,6 +9,7 @@ import (
"github.com/eyebluecn/tank/code/tool/third"
"github.com/eyebluecn/tank/code/tool/util"
"github.com/eyebluecn/tank/code/tool/uuid"
"github.com/glebarez/sqlite"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
@ -99,6 +100,7 @@ func (this *InstallController) RegisterRoutes() map[string]func(writer http.Resp
}
func (this *InstallController) openDbConnection(writer http.ResponseWriter, request *http.Request) *gorm.DB {
dbType := request.FormValue("dbType")
mysqlPortStr := request.FormValue("mysqlPort")
mysqlHost := request.FormValue("mysqlHost")
mysqlSchema := request.FormValue("mysqlSchema")
@ -113,28 +115,47 @@ func (this *InstallController) openDbConnection(writer http.ResponseWriter, requ
mysqlPort = tmp
}
mysqlUrl := util.GetMysqlUrl(mysqlPort, mysqlHost, mysqlSchema, mysqlUsername, mysqlPassword, mysqlCharset)
this.logger.Info("Connect MySQL %s", mysqlUrl)
//log config
dbLogger := logger.New(
log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer
logger.Config{
SlowThreshold: time.Second, // slow SQL 1s
LogLevel: logger.Info, // log level
IgnoreRecordNotFoundError: true, // ignore ErrRecordNotFound
Colorful: false, // colorful print
SlowThreshold: time.Second, // slow SQL 1s
LogLevel: logger.Silent, // log level. open when debug.
IgnoreRecordNotFoundError: true, // ignore ErrRecordNotFound
Colorful: false, // colorful print
},
)
//table name strategy
namingStrategy := core.CONFIG.NamingStrategy()
var err error = nil
db, err := gorm.Open(mysql.Open(mysqlUrl), &gorm.Config{Logger: dbLogger, NamingStrategy: namingStrategy})
this.PanicError(err)
if dbType == "sqlite" {
return db
var err error = nil
sqliteFolder := core.CONFIG.SqliteFolder() + "/tank.sqlite"
this.logger.Info("Connect Sqlite %s", sqliteFolder)
db, err := gorm.Open(sqlite.Open(sqliteFolder), &gorm.Config{Logger: dbLogger, NamingStrategy: namingStrategy})
if err != nil {
core.LOGGER.Panic("failed to connect sqlite database")
}
//sqlite lock issue. https://gist.github.com/mrnugget/0eda3b2b53a70fa4a894
phyDb, err := db.DB()
phyDb.SetMaxOpenConns(1)
return db
} else {
mysqlUrl := util.GetMysqlUrl(mysqlPort, mysqlHost, mysqlSchema, mysqlUsername, mysqlPassword, mysqlCharset)
this.logger.Info("Connect MySQL %s", mysqlUrl)
var err error = nil
db, err := gorm.Open(mysql.Open(mysqlUrl), &gorm.Config{Logger: dbLogger, NamingStrategy: namingStrategy})
this.PanicError(err)
return db
}
}
@ -181,10 +202,23 @@ func (this *InstallController) getTableMeta(gormDb *gorm.DB, entity interface{})
for _, field := range allFields {
//tag with `gorm:"-"` will be ""
if field.Name != "" {
database := gormDb.Migrator().CurrentDatabase()
if !third.MysqlMigratorHasColumn(gormDb, database, tableName, field.Name) {
missingFields = append(missingFields, field)
//if sqlite
if _, ok := gormDb.Dialector.(*sqlite.Dialector); ok {
if !gormDb.Migrator().HasColumn(tableName, field.Name) {
missingFields = append(missingFields, field)
}
} else {
if !third.MysqlMigratorHasColumn(gormDb, database, tableName, field.Name) {
missingFields = append(missingFields, field)
}
}
}
}
@ -258,16 +292,24 @@ func (this *InstallController) TableInfoList(writer http.ResponseWriter, request
func (this *InstallController) CreateTable(writer http.ResponseWriter, request *http.Request) *result.WebResult {
var installTableInfos []*InstallTableInfo
mysqlCharset := request.FormValue("mysqlCharset")
db := this.openDbConnection(writer, request)
defer this.closeDbConnection(db)
for _, iBase := range this.tableNames {
//complete the missing fields or create table. use utf8 charset
err := db.Set("gorm:table_options", fmt.Sprintf("CHARSET=%s", mysqlCharset)).AutoMigrate(iBase)
this.PanicError(err)
if _, ok := db.Dialector.(*sqlite.Dialector); ok {
//if sqlite. no need to set CHARSET.
err := db.AutoMigrate(iBase)
this.PanicError(err)
} else {
//use utf8 charset
mysqlCharset := request.FormValue("mysqlCharset")
err := db.Set("gorm:table_options", fmt.Sprintf("CHARSET=%s", mysqlCharset)).AutoMigrate(iBase)
this.PanicError(err)
}
//complete the missing fields or create table.
tableName, exist, allFields, missingFields := this.getTableMeta(db, iBase)
installTableInfos = append(installTableInfos, &InstallTableInfo{
Name: tableName,
@ -383,6 +425,7 @@ func (this *InstallController) ValidateAdmin(writer http.ResponseWriter, request
//Finish the installation
func (this *InstallController) Finish(writer http.ResponseWriter, request *http.Request) *result.WebResult {
dbType := request.FormValue("dbType")
mysqlPortStr := request.FormValue("mysqlPort")
mysqlHost := request.FormValue("mysqlHost")
mysqlSchema := request.FormValue("mysqlSchema")
@ -414,7 +457,7 @@ func (this *InstallController) Finish(writer http.ResponseWriter, request *http.
}
//announce the config to write config to tank.json
core.CONFIG.FinishInstall(mysqlPort, mysqlHost, mysqlSchema, mysqlUsername, mysqlPassword, mysqlCharset)
core.CONFIG.FinishInstall(dbType, mysqlPort, mysqlHost, mysqlSchema, mysqlUsername, mysqlPassword, mysqlCharset)
//announce the context to broadcast the installation news to bean.
core.CONTEXT.InstallOk()

8
code/rest/matter_model.go
View File

@ -34,8 +34,8 @@ type Matter struct {
Sort int64 `json:"sort" gorm:"type:bigint(20) not null"`
UpdateTime time.Time `json:"updateTime" gorm:"type:timestamp not null;default:CURRENT_TIMESTAMP"`
CreateTime time.Time `json:"createTime" gorm:"type:timestamp not null;default:'2018-01-01 00:00:00'"`
Puuid string `json:"puuid" gorm:"type:char(36);index:idx_puuid"`
UserUuid string `json:"userUuid" gorm:"type:char(36);index:idx_uu"`
Puuid string `json:"puuid" gorm:"type:char(36);index:idx_matter_puuid"` //index should unique globally.
UserUuid string `json:"userUuid" gorm:"type:char(36);index:idx_matter_uu"`
Username string `json:"username" gorm:"type:varchar(45) not null"`
Dir bool `json:"dir" gorm:"type:tinyint(1) not null;default:0"`
Name string `json:"name" gorm:"type:varchar(255) not null"`
@ -48,8 +48,8 @@ type Matter struct {
Children []*Matter `json:"-" gorm:"-"`
Prop string `json:"prop" gorm:"type:varchar(1024) not null;default:'{}'"`
VisitTime time.Time `json:"visitTime" gorm:"type:timestamp not null;default:'2018-01-01 00:00:00'"`
Deleted bool `json:"deleted" gorm:"type:tinyint(1) not null;index:idx_del;default:0"`
DeleteTime time.Time `json:"deleteTime" gorm:"type:timestamp not null;index:idx_delt;default:'2018-01-01 00:00:00'"`
Deleted bool `json:"deleted" gorm:"type:tinyint(1) not null;index:idx_matter_del;default:0"`
DeleteTime time.Time `json:"deleteTime" gorm:"type:timestamp not null;index:idx_matter_delt;default:'2018-01-01 00:00:00'"`
}
// get matter's absolute path. the Path property is relative path in db.

86
code/support/tank_config.go
View File

@ -20,6 +20,8 @@ type TankConfig struct {
matterPath string
//mysql url.
mysqlUrl string
//sqlite file path
sqliteFolder string
//configs in tank.json
item *ConfigItem
}
@ -30,7 +32,10 @@ type ConfigItem struct {
ServerPort int
//file storage location. eg./var/www/matter
MatterPath string
//mysql configurations.
//********db configurations.********
//default value is "mysql"
DbType string
//********mysql configurations..********
//mysql port
MysqlPort int
//mysql host
@ -43,6 +48,9 @@ type ConfigItem struct {
MysqlPassword string
//mysql charset
MysqlCharset string
//********sqlite configurations..********
//default value is matter/
SqliteFolder string
}
//validate whether the config file is ok
@ -53,33 +61,39 @@ func (this *ConfigItem) validate() bool {
return false
}
if this.MysqlUsername == "" {
core.LOGGER.Error("MysqlUsername is not configured")
return false
}
if this.DbType == "sqlite" {
if this.MysqlPassword == "" {
core.LOGGER.Error("MysqlPassword is not configured")
return false
}
} else {
if this.MysqlHost == "" {
core.LOGGER.Error("MysqlHost is not configured")
return false
}
if this.MysqlUsername == "" {
core.LOGGER.Error("MysqlUsername is not configured")
return false
}
if this.MysqlPort == 0 {
core.LOGGER.Error("MysqlPort is not configured")
return false
}
if this.MysqlPassword == "" {
core.LOGGER.Error("MysqlPassword is not configured")
return false
}
if this.MysqlSchema == "" {
core.LOGGER.Error("MysqlSchema is not configured")
return false
}
if this.MysqlCharset == "" {
core.LOGGER.Error("MysqlCharset is not configured")
return false
if this.MysqlHost == "" {
core.LOGGER.Error("MysqlHost is not configured")
return false
}
if this.MysqlPort == 0 {
core.LOGGER.Error("MysqlPort is not configured")
return false
}
if this.MysqlSchema == "" {
core.LOGGER.Error("MysqlSchema is not configured")
return false
}
if this.MysqlCharset == "" {
core.LOGGER.Error("MysqlCharset is not configured")
return false
}
}
return true
@ -169,11 +183,30 @@ func (this *TankConfig) ServerPort() int {
return this.serverPort
}
//get the db type
func (this *TankConfig) DbType() string {
return this.item.DbType
}
//mysql url
func (this *TankConfig) MysqlUrl() string {
return this.mysqlUrl
}
//get the sqlite path
func (this *TankConfig) SqliteFolder() string {
if this.sqliteFolder == "" {
//use default file location.
if this.item == nil || this.item.SqliteFolder == "" {
this.sqliteFolder = util.GetHomePath() + "/matter"
} else {
this.sqliteFolder = util.UniformPath(this.item.SqliteFolder)
}
}
return this.sqliteFolder
}
//matter path
func (this *TankConfig) MatterPath() string {
return this.matterPath
@ -187,10 +220,11 @@ func (this *TankConfig) NamingStrategy() schema.NamingStrategy {
}
}
//Finish the installation. Write config to tank.json
func (this *TankConfig) FinishInstall(mysqlPort int, mysqlHost string, mysqlSchema string, mysqlUsername string, mysqlPassword string, mysqlCharset string) {
//TODO: Finish the installation. Write config to tank.json. add sqlite support.
func (this *TankConfig) FinishInstall(dbType string, mysqlPort int, mysqlHost string, mysqlSchema string, mysqlUsername string, mysqlPassword string, mysqlCharset string) {
var configItem = &ConfigItem{
DbType: dbType,
//server port
ServerPort: core.CONFIG.ServerPort(),
//file storage location. eg./var/www/matter

28
code/support/tank_context.go
View File

@ -4,6 +4,7 @@ import (
"github.com/eyebluecn/tank/code/core"
"github.com/eyebluecn/tank/code/rest"
"github.com/eyebluecn/tank/code/tool/cache"
"github.com/glebarez/sqlite"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
@ -80,19 +81,36 @@ func (this *TankContext) OpenDb() {
log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer
logger.Config{
SlowThreshold: time.Second, // slow SQL 1s
LogLevel: logger.Silent, // log level
LogLevel: logger.Silent, // log level. open when debug.
IgnoreRecordNotFoundError: true, // ignore ErrRecordNotFound
Colorful: false, // colorful print
},
)
//table name strategy.
namingStrategy := core.CONFIG.NamingStrategy()
var err error = nil
this.db, err = gorm.Open(mysql.Open(core.CONFIG.MysqlUrl()), &gorm.Config{Logger: dbLogger, NamingStrategy: namingStrategy})
if core.CONFIG.DbType() == "sqlite" {
if err != nil {
core.LOGGER.Panic("failed to connect mysql database")
var err error = nil
this.db, err = gorm.Open(sqlite.Open(core.CONFIG.SqliteFolder()+"/tank.sqlite"), &gorm.Config{Logger: dbLogger, NamingStrategy: namingStrategy})
if err != nil {
core.LOGGER.Panic("failed to connect mysql database")
}
//sqlite lock issue. https://gist.github.com/mrnugget/0eda3b2b53a70fa4a894
phyDb, err := this.db.DB()
phyDb.SetMaxOpenConns(1)
} else {
var err error = nil
this.db, err = gorm.Open(mysql.Open(core.CONFIG.MysqlUrl()), &gorm.Config{Logger: dbLogger, NamingStrategy: namingStrategy})
if err != nil {
core.LOGGER.Panic("failed to connect mysql database")
}
}
}