From 277b4b9f22f5d06f97bb61415c0b4353a135f5a6 Mon Sep 17 00:00:00 2001 From: Evan Chen Date: Thu, 16 Dec 2021 12:11:33 +0800 Subject: [PATCH] update --- .gitignore | 1 + Makefile | 8 ++ README.md | 3 + auth/access.go | 52 ++++++++ auth/api_auth.go | 88 +++++++++++++ auth/api_grp.go | 187 ++++++++++++++++++++++++++ auth/api_user.go | 190 +++++++++++++++++++++++++++ auth/auth.go | 74 +++++++++++ auth/errors.go | 49 +++++++ auth/group.go | 30 +++++ auth/helper.go | 201 ++++++++++++++++++++++++++++ auth/jwt.go | 193 +++++++++++++++++++++++++++ auth/service.go | 81 ++++++++++++ auth/user.go | 72 ++++++++++ docs/API.md | 0 errors/errors.go | 39 ++++++ errors/net_errors.go | 27 ++++ go.mod | 58 ++++++++ go.sum | 292 +++++++++++++++++++++++++++++++++++++++++ main.go | 23 ++++ scripts/build.sh | 0 server/helper.go | 47 +++++++ server/middleware.go | 56 ++++++++ server/server.go | 12 ++ server/service.go | 78 +++++++++++ store/db.go | 194 +++++++++++++++++++++++++++ system/helper.go | 145 ++++++++++++++++++++ system/helper_test.go | 89 +++++++++++++ system/profile.go | 38 ++++++ system/profile_test.go | 33 +++++ system/setup.go | 127 ++++++++++++++++++ system/system.go | 222 +++++++++++++++++++++++++++++++ system/system_test.go | 24 ++++ task/logger.go | 39 ++++++ task/profile.go | 65 +++++++++ task/service.go | 23 ++++ task/store.go | 108 +++++++++++++++ task/task.go | 93 +++++++++++++ task/task_test.go | 124 +++++++++++++++++ util/logging.go | 93 +++++++++++++ util/util.go | 21 +++ 41 files changed, 3299 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 auth/access.go create mode 100644 auth/api_auth.go create mode 100644 auth/api_grp.go create mode 100644 auth/api_user.go create mode 100644 auth/auth.go create mode 100644 auth/errors.go create mode 100644 auth/group.go create mode 100644 auth/helper.go create mode 100644 auth/jwt.go create mode 100644 auth/service.go create mode 100644 auth/user.go create mode 100644 docs/API.md create mode 100644 errors/errors.go create mode 100644 errors/net_errors.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 scripts/build.sh create mode 100644 server/helper.go create mode 100644 server/middleware.go create mode 100644 server/server.go create mode 100644 server/service.go create mode 100644 store/db.go create mode 100644 system/helper.go create mode 100644 system/helper_test.go create mode 100644 system/profile.go create mode 100644 system/profile_test.go create mode 100644 system/setup.go create mode 100644 system/system.go create mode 100644 system/system_test.go create mode 100644 task/logger.go create mode 100644 task/profile.go create mode 100644 task/service.go create mode 100644 task/store.go create mode 100644 task/task.go create mode 100644 task/task_test.go create mode 100644 util/logging.go create mode 100644 util/util.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a340c10 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +work \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..330dc96 --- /dev/null +++ b/Makefile @@ -0,0 +1,8 @@ +run: + APP_SERVER_PORT=8000 \ + APP_LOG_LEVEL=-1 \ + APP_PROD=false \ + APP_LOG_PRETTY=true \ + APP_DB_TYPE=sqlite \ + APP_DATA=work \ + go run main.go \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..b126c23 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Kumoly App + +combine all needed module into one lib \ No newline at end of file diff --git a/auth/access.go b/auth/access.go new file mode 100644 index 0000000..6d6ac75 --- /dev/null +++ b/auth/access.go @@ -0,0 +1,52 @@ +package auth + +import ( + "github.com/gin-gonic/gin" + "kumoly.io/kumoly/app/errors" +) + +const ( + SYS_AUTH_PREFIX = "*" + SYSTEM = SYS_AUTH_PREFIX + "system" + ADMIN = SYS_AUTH_PREFIX + "admin" + USER = SYS_AUTH_PREFIX + "user" +) + +// ACHas access control has returns if the user is in group +func ACHas(c *gin.Context, grps ...string) bool { + cliams, err := GetContextClaims(c) + if err != nil { + return false + } + return cliams.HasGroup(grps...) +} + +// ACMust access control as middleware, panics if not in group +func ACMust(grps ...string) func(c *gin.Context) { + return func(c *gin.Context) { + cliams, err := GetContextClaims(c) + if err != nil { + panic(err) + } + if cliams.HasGroup(grps...) { + c.Next() + } else { + panic(errors.ErrorForbidden) + } + } +} + +// ACSystem shorthand for ACMust(SYSTEM) +func ACSystem() func(c *gin.Context) { + return ACMust(SYSTEM) +} + +// ACAdmin shorthand for ACMust(ADMIN) +func ACAdmin() func(c *gin.Context) { + return ACMust(ADMIN) +} + +// ACUser shorthand for ACMust(USER) +func ACUser() func(c *gin.Context) { + return ACMust(USER) +} diff --git a/auth/api_auth.go b/auth/api_auth.go new file mode 100644 index 0000000..2c42e1f --- /dev/null +++ b/auth/api_auth.go @@ -0,0 +1,88 @@ +package auth + +import ( + "encoding/base64" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/rs/zerolog/log" + "kumoly.io/kumoly/app/errors" + "kumoly.io/kumoly/app/server" +) + +type apiLoginReq struct { + Name string `json:"username" example:"user" binding:"required"` + // Email string `json:"email" example:"user@example.com" binding:"required,email"` + PwdB64 string `json:"password" example:"YWRtaW4=" binding:"required,base64"` +} + +func ApiLogin(c *gin.Context) { + var data apiLoginReq + if err := c.ShouldBindJSON(&data); err != nil { + panic(err) + } + pwd, err := base64.URLEncoding.DecodeString(data.PwdB64) + if err != nil { + panic(err) + } + usr := &User{} + err = DB.Preload("Profile").Preload("Groups").Where("username = ?", data.Name).First(usr).Error + if err != nil { + panic(ErrorLoginFailed) + } + err = usr.ValidatePassword(string(pwd)) + if err != nil { + log.Error().Str("mod", "auth").Err(err).Msg("wrong password") + usr.LoginFailed += 1 + DB.Model(&usr).Update("login_failed", usr.LoginFailed) + panic(ErrorLoginFailed) + } + if usr.SSOEnabled { + panic(ErrorUserIsSSO) + } + if !usr.Activated { + log.Error().Str("mod", "auth"). + Err(ErrorUserNotActivated). + Str("user", usr.Username).Str("uid", usr.ID). + Msg("not activated") + panic(ErrorUserNotActivated) + } + grps := make([]string, len(usr.Groups)) + for i, g := range usr.Groups { + grps[i] = g.Name + } + err = SetClaims(c, &Claims{ + Uid: usr.ID, + Groups: grps, + }) + if err != nil { + panic(err) + } + usr.LastLogin = time.Now() + usr.LastLoginIP = c.ClientIP() + usr.LoginFailed = 0 + DB.Model(&usr).Updates(map[string]interface{}{ + "last_login": usr.LastLogin, + "login_failed": usr.LoginFailed, + "last_login_ip": usr.LastLoginIP, + }) + server.Res(c, &server.Response{ + Status: 200, + Data: usr, + }) +} + +func ApiLogout(c *gin.Context) { + ClearToken(c) + server.Res(c, &server.Response{Data: "ok"}) +} + +func ApiMe(c *gin.Context) { + usr, err := GetUser(c, DB) + if err != nil { + ClearToken(c) + panic(errors.NewError(http.StatusUnauthorized, err)) + } + server.Res(c, &server.Response{Data: usr}) +} diff --git a/auth/api_grp.go b/auth/api_grp.go new file mode 100644 index 0000000..269bcc5 --- /dev/null +++ b/auth/api_grp.go @@ -0,0 +1,187 @@ +package auth + +import ( + "strings" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" + "kumoly.io/kumoly/app/errors" + "kumoly.io/kumoly/app/server" +) + +func ApiGrps(c *gin.Context) { + grps := []Group{} + var result *gorm.DB + if ACHas(c, SYSTEM) { + result = DB.Find(&grps) + } else { + result = DB.Where("name not like ?", SYS_AUTH_PREFIX+"%").Find(&grps) + } + if result.Error != nil { + panic(result.Error) + } + server.Res(c, &server.Response{Data: grps}) +} + +type apiGrpNewReq struct { + Name string `json:"name" example:"user" binding:"required"` + DisplayName string `json:"display_name"` + Description string `json:"description"` +} + +func ApiGrpNew(c *gin.Context) { + var data apiGrpNewReq + if err := c.ShouldBindJSON(&data); err != nil { + panic(err) + } + if strings.HasPrefix(data.Name, SYS_AUTH_PREFIX) && !ACHas(c, ADMIN) { + panic(errors.ErrorForbidden) + } + grp := &Group{ + Name: data.Name, + Description: data.Description, + DisplayName: data.DisplayName, + } + if err := DB.Create(grp).Error; err != nil { + panic(err) + } + server.Res(c, &server.Response{Data: grp}) +} + +type apiGrpUpdateReq struct { + ID uint `json:"id" binding:"required"` + Name string `json:"name" binding:"required"` + DisplayName string `json:"display_name"` + Description string `json:"description"` +} + +func ApiGrpUpdate(c *gin.Context) { + fetch := c.Query("fetch") + var data apiGrpUpdateReq + if err := c.ShouldBindJSON(&data); err != nil { + panic(err) + } + grp := &Group{} + err := DB.First(grp, data.ID).Error + if err != nil { + panic(errors.NewError(404, err)) + } + if strings.HasPrefix(grp.Name, SYS_AUTH_PREFIX) { + panic(errors.ErrorForbidden) + } + if !ACHas(c, ADMIN, grp.Name) { + panic(errors.ErrorForbidden) + } + result := DB.Model(&grp).Updates(map[string]interface{}{ + "name": data.Name, + "display_name": data.DisplayName, + "description": data.Description, + }) + if result.Error != nil { + panic(result.Error) + } + if fetch != "" { + DB.Preload("Users").First(grp, data.ID) + } + server.Res(c, &server.Response{Data: grp}) +} + +func ApiGrpDel(c *gin.Context) { + id := c.Param("id") + if id == "" { + panic(errors.ErrorBadRequest) + } + grp := &Group{} + err := DB.First(grp, id).Error + if err != nil { + panic(errors.NewError(404, err)) + } + if strings.HasPrefix(grp.Name, SYS_AUTH_PREFIX) && !ACHas(c, ADMIN) { + panic(errors.ErrorForbidden) + } + err = DB.Transaction(func(tx *gorm.DB) error { + err = tx.Model(&grp).Association("Users").Clear() + if err != nil { + return err + } + err = tx.Delete(grp).Error + if err != nil { + return err + } + return nil + }) + if err != nil { + panic(err) + } + server.Res(c, &server.Response{Data: "ok"}) +} + +func ApiGrpAssign(c *gin.Context) { + uid := c.Param("uid") + gid := c.Param("gid") + if uid == "" || gid == "" { + panic(errors.ErrorBadRequest) + } + usr := &User{} + err := DB.Where("id = ?", uid).First(usr).Error + if err != nil { + panic(errors.NewError(404, err)) + } + grp := &Group{} + err = DB.First(grp, gid).Error + if err != nil { + panic(errors.NewError(404, err)) + } + + // deny access + if !ACHas(c, grp.Name, ADMIN, SYSTEM) { + panic(errors.ErrorForbidden) + } + + // only sys can add sys + if grp.Name == SYSTEM { + if !ACHas(c, SYSTEM) { + panic(errors.ErrorForbidden) + } + } + err = DB.Transaction(func(tx *gorm.DB) error { + return tx.Model(usr).Association("Groups").Append(grp) + }) + if err != nil { + panic(err) + } + server.Res(c, &server.Response{Data: "ok"}) +} + +func ApiGrpRemove(c *gin.Context) { + uid := c.Param("uid") + gid := c.Param("gid") + if uid == "" || gid == "" { + panic(errors.ErrorBadRequest) + } + usr := &User{} + err := DB.Where("id = ?", uid).First(usr).Error + if err != nil { + panic(errors.NewError(404, err)) + } + grp := &Group{} + err = DB.First(grp, gid).Error + if err != nil { + panic(errors.NewError(404, err)) + } + if grp.Name == SYSTEM { + if !ACHas(c, SYSTEM) { + panic(errors.ErrorForbidden) + } + } + if grp.Name == ADMIN && IsLastAdmin() { + panic(ErrorDelLastAdmin) + } + err = DB.Transaction(func(tx *gorm.DB) error { + return tx.Model(usr).Association("Groups").Delete(grp) + }) + if err != nil { + panic(err) + } + server.Res(c, &server.Response{Data: "ok"}) +} diff --git a/auth/api_user.go b/auth/api_user.go new file mode 100644 index 0000000..1ba8e82 --- /dev/null +++ b/auth/api_user.go @@ -0,0 +1,190 @@ +package auth + +import ( + "encoding/base64" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" + "kumoly.io/kumoly/app/errors" + "kumoly.io/kumoly/app/server" +) + +func ApiUsers(c *gin.Context) { + users := []User{} + result := DB.Preload("Groups").Preload("Profile").Find(&users) + if result.Error != nil { + panic(result.Error) + } + server.Res(c, &server.Response{Data: users}) +} + +type apiUserNewReq struct { + Name string `json:"username" example:"user" binding:"required"` + PwdB64 string `json:"password" example:"YWRtaW4=" binding:"required,base64"` +} + +func ApiUserNew(c *gin.Context) { + var data apiUserNewReq + if err := c.ShouldBindJSON(&data); err != nil { + panic(err) + } + pwd, err := base64.URLEncoding.DecodeString(data.PwdB64) + if err != nil { + panic(err) + } + usr := &User{ + Username: data.Name, + Password: string(pwd), + } + err = NewUser(usr, DB) + if err != nil { + panic(err) + } + server.Res(c, &server.Response{Data: usr}) +} + +func ApiUserDelete(c *gin.Context) { + var err error + id := c.Param("id") + if id == "" { + panic(errors.ErrorBadRequest) + } + usr := &User{ + ID: id, + } + err = DB.First(usr, "id = ?", id).Error + if err != nil { + panic(errors.NewError(404, err)) + } + + if IsLastAdminUser(id) { + panic(ErrorDelLastAdmin) + } + + prof := &Profile{} + profExist := true + err = DB.First(prof, "user_id = ?", id).Error + if err != nil { + profExist = false + } + + grp := &Group{} + grpExist := false + err = DB.Where("name = ?", usr.Username).First(grp).Error + if err == nil { + grpExist = true + } + err = DB.Transaction(func(tx *gorm.DB) error { + if grpExist { + err = tx.Model(&grp).Association("Users").Clear() + if err != nil { + return err + } + } + err = tx.Model(&usr).Association("Groups").Clear() + if err != nil { + return err + } + err = tx.Model(&usr).Association("Profile").Clear() + if err != nil { + return err + } + + if profExist { + err = tx.Delete(&prof).Error + if err != nil { + return err + } + } + err = tx.Delete(usr).Error + if err != nil { + return err + } + err = tx.Delete(grp).Error + return err + }) + if err != nil { + panic(err) + } + server.Res(c, &server.Response{Data: "ok"}) +} + +type apiUserChangePasswdReq struct { + UserID string `json:"uid" example:"c35icut3sbav3gg5j1b0" binding:"required"` + PwdB64 string `json:"old" ` + NewPwdB64 string `json:"new" example:"YWRtaW4=" binding:"required,base64"` +} + +func ApiUserChangePasswd(c *gin.Context) { + var err error + var data apiUserChangePasswdReq + if err = c.ShouldBindJSON(&data); err != nil { + panic(err) + } + usr := &User{} + err = DB.First(usr, "id = ?", data.UserID).Error + if err != nil { + panic(errors.NewError(404, err)) + } + + if !ACHas(c, ADMIN) { + if !ACHas(c, usr.Username) { + panic(errors.ErrorForbidden) + } + old_pwd, err := base64.URLEncoding.DecodeString(data.PwdB64) + if err != nil { + panic(err) + } + err = usr.ValidatePassword(string(old_pwd)) + if err != nil { + panic(err) + } + } + + new_pwd, err := base64.URLEncoding.DecodeString(data.NewPwdB64) + if err != nil { + panic(err) + } + err = usr.ChangePassword(DB, string(new_pwd)) + if err != nil { + panic(err) + } + server.Res(c, &server.Response{Data: "ok"}) +} + +func ApiUserActivate(c *gin.Context) { + usr := &User{} + uid := c.Param("id") + if uid == "" { + panic(errors.ErrorBadRequest) + } + err := DB.Where("id = ?", uid).First(usr).Error + if err != nil { + panic(errors.NewError(404, err)) + } + err = DB.Model(&usr).Update("activated", true).Error + if err != nil { + panic(err) + } + server.Res(c, &server.Response{Data: "ok"}) +} + +func ApiUserDeactivate(c *gin.Context) { + usr := &User{} + uid := c.Param("id") + if uid == "" { + panic(errors.ErrorBadRequest) + } + err := DB.Where("id = ?", uid).First(usr).Error + if err != nil { + panic(errors.NewError(404, err)) + } + if IsLastAdminUser(uid) { + panic(ErrorDelLastAdmin) + } + err = DB.Model(&usr).Update("activated", false).Error + if err != nil { + panic(err) + } + server.Res(c, &server.Response{Data: "ok"}) +} diff --git a/auth/auth.go b/auth/auth.go new file mode 100644 index 0000000..1367115 --- /dev/null +++ b/auth/auth.go @@ -0,0 +1,74 @@ +package auth + +import ( + "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt" + "github.com/spf13/viper" + "kumoly.io/kumoly/app/system" +) + +func init() { + // secret for jwt + viper.SetDefault("auth.secret", "secret") + + // expires in sec, '0' to indicate token will not expire + viper.SetDefault("auth.expire", 1800) + + // default admin user + viper.SetDefault("admin.user", "admin") + viper.SetDefault("admin.passwd", "admin") + +} + +var std *Auth + +func Setup() { + std = NewAuth() +} + +func Injector(router *gin.RouterGroup) *system.Inject { + return std.Injector(router) +} + +// Parse tok str to token object +func Parse(tok string) (token *jwt.Token, err error) { + return std.Parse(tok) +} + +// ParseClaims parse token string to claims object +func ParseClaims(tok string) (claims *Claims, err error) { + return std.ParseClaims(tok) +} + +// SetToken in header and cookie(if CookieMode) +func SetToken(c *gin.Context, tok string) { + std.SetToken(c, tok) +} + +// SetClaims directly from http request +func SetClaims(c *gin.Context, claims *Claims) error { + return std.SetClaims(c, claims) +} + +// GetToken from header or cookie +func GetToken(c *gin.Context) (tok string, err error) { + return std.GetToken(c) +} + +// GetClaims directly from http request +func GetClaims(c *gin.Context) (claims *Claims, err error) { + return std.GetClaims(c) +} + +func ClearToken(c *gin.Context) { + std.ClearToken(c) +} + +// New tok str +func NewToken(claims Claims) (tok string, err error) { + return std.NewToken(claims) +} + +func Middleware(c *gin.Context) { + std.Middleware(c) +} diff --git a/auth/errors.go b/auth/errors.go new file mode 100644 index 0000000..3010b28 --- /dev/null +++ b/auth/errors.go @@ -0,0 +1,49 @@ +package auth + +import ( + "net/http" + + "kumoly.io/kumoly/app/errors" +) + +var ErrorLoginFailed = errors.Error{ + Code: http.StatusUnauthorized, + ID: "ErrorLoginFailed", + Message: "Wrong username or password.", +} + +var ErrorUserNotActivated = errors.Error{ + Code: http.StatusForbidden, + ID: "ErrorUserNotActivated", + Message: "User is not activated.", +} + +var ErrorUserIsSSO = errors.Error{ + Code: http.StatusForbidden, + ID: "ErrorUserIsSSO", + Message: "user is sso enabled.", +} + +var ErrorTokenNotValid = errors.Error{ + Code: http.StatusUnauthorized, + ID: "ErrorTokenNotValid", + Message: "token not valid", +} + +var ErrorUnknownClaims = errors.Error{ + Code: http.StatusUnauthorized, + ID: "ErrorUnknownClaims", + Message: "unknown claims", +} + +var ErrorDelLastAdmin = errors.Error{ + Code: http.StatusForbidden, + ID: "ErrorDelLastAdmin", + Message: "Cannot remove last admin account.", +} + +var ErrorBadRequestTmpl = errors.Error{ + Code: http.StatusBadRequest, + ID: "ErrorBadRequest", + Tmpl: "%v is not sufficient.", +} diff --git a/auth/group.go b/auth/group.go new file mode 100644 index 0000000..3d625ee --- /dev/null +++ b/auth/group.go @@ -0,0 +1,30 @@ +package auth + +import ( + "strings" + "time" + + "gorm.io/gorm" +) + +//Group enum of user permission groups +type Group struct { + ID uint `gorm:"primaryKey"` + CreatedAt time.Time + UpdatedAt time.Time + + // Name starting with '*' is reserved and starting with '_' is internal + Name string `gorm:"unique;not null"` + DisplayName string + Description string + Users []*User `gorm:"many2many:user_groups;"` +} + +func (grp *Group) BeforeSave(tx *gorm.DB) (err error) { + + // set displayname + if grp.DisplayName == "" { + grp.DisplayName = strings.TrimPrefix(grp.Name, "*") + } + return +} diff --git a/auth/helper.go b/auth/helper.go new file mode 100644 index 0000000..9e07e06 --- /dev/null +++ b/auth/helper.go @@ -0,0 +1,201 @@ +package auth + +import ( + "github.com/gin-gonic/gin" + "golang.org/x/crypto/bcrypt" + "gorm.io/gorm" +) + +// HasGroup checks if a group is in the claim groups +func (c *Claims) HasGroup(grps ...string) bool { + m := make(map[string]bool) + for _, grp := range grps { + m[grp] = true + } + for _, trg := range c.Groups { + if m[trg] { + return true + } + } + return false +} + +func (srv Service) SetDefaultGroups() error { + srv.Logger.Debug().Msg("Setup default groups") + for _, g := range []string{SYSTEM, ADMIN, USER} { + grp := &Group{} + if err := DB.Where("name = ?", g).First(grp).Error; err != nil { + err := DB.Create(&Group{ + Name: g, + }).Error + if err != nil { + srv.Logger.Error().Err(err).Msg("create group error") + return err + } + } + } + return nil +} + +func (srv Service) SetDefaultAdmin(username, password string) error { + admin := &Group{} + err := DB.Where("name = ?", ADMIN).First(admin).Error + if err != nil { + srv.Logger.Error().Err(err).Msg("SetDefaultAdmin") + return err + } + usrgrp := struct { + GroupID uint + UserID string + }{} + result := DB. + Raw("select * from user_groups where group_id = ?", admin.ID). + Scan(&usrgrp) + if result.Error != nil { + srv.Logger.Error().Err(result.Error).Msg("SetDefaultAdmin") + return result.Error + } + usr := &User{} + if result.RowsAffected == 0 { + srv.Logger.Debug().Msg("Setting up admin account") + pwd, _ := bcrypt.GenerateFromPassword([]byte(password), 14) + usr.Username = username + usr.Password = string(pwd) + err = DB.Transaction(func(tx *gorm.DB) error { + err := tx.Create(usr).Error + if err != nil { + return err + } + grp := &Group{ + Name: username, + } + err = tx.Create(grp).Error + if err != nil { + return err + } + err = tx.Model(usr).Association("Groups").Append(admin, grp) + if err != nil { + return err + } + profile := &Profile{ + DisplayName: username, + } + err = tx.Model(usr).Association("Profile").Append(profile) + if err != nil { + return err + } + return nil + }) + if err != nil { + return err + } + SetGroup(usr.ID, USER, true) + } + return nil +} + +func IsLastAdmin() bool { + var count int + DB.Raw(` +select count(*) from user_groups where group_id = ( + select id from groups where name = ? +) +`, ADMIN).Scan(&count) + return count == 1 +} + +func IsLastAdminUser(uid string) bool { + var count int + DB.Raw(` +select count(*) from user_groups ug, users u +where group_id = (select id from groups where name = ?) +and user_id <> ? +and u.id = user_id +and u.activated = 1 +`, ADMIN, uid).Scan(&count) + return count == 0 +} + +func SetAdmin(uid string, set bool) { + SetGroup(uid, ADMIN, set) +} + +func SetGroup(uid string, group_name string, set bool) { + var grp_id uint + DB.Raw( + `select id from groups where name = ?`, + group_name, + ).Scan(&grp_id) + var count int + DB.Raw(` +select count(*) from user_groups where group_id = ? +and user_id = ?`, + grp_id, uid).Scan(&count) + + // remove + if count == 1 && !set { + DB.Exec(`delete from user_groups where group_id = ? + and user_id = ?`, grp_id, uid) + } + // add + if count == 0 && set { + DB.Exec(`insert into user_groups (group_id, user_id) + values (?, ?)`, grp_id, uid) + } +} + +func GetUser(c *gin.Context, db *gorm.DB) (*User, error) { + claim, err := GetContextClaims(c) + if err != nil { + return nil, err + } + usr := &User{} + err = db.Preload("Groups").Preload("Profile").Where("id = ?", claim.Uid).First(usr).Error + if err != nil { + return nil, err + } + return usr, nil +} + +// NewUser the password is still not hashed +func NewUser(usr *User, db *gorm.DB) error { + if usr.Username == "" || usr.Password == "" { + return ErrorBadRequestTmpl.New("auth.User") + } + bytes, err := bcrypt.GenerateFromPassword([]byte(usr.Password), 14) + if err != nil { + return err + } + usr.Password = string(bytes) + err = DB.Transaction(func(tx *gorm.DB) error { + err := tx.Create(usr).Error + if err != nil { + return err + } + grp := &Group{ + Name: usr.Username, + } + err = tx.Create(grp).Error + if err != nil { + return err + } + grp_user := &Group{} + err = DB.Where("name = ?", USER).First(grp_user).Error + if err != nil { + return err + } + err = tx.Model(usr).Association("Groups").Append(grp, grp_user) + if err != nil { + return err + } + profile := &Profile{ + DisplayName: usr.Username, + } + err = tx.Model(usr).Association("Profile").Append(profile) + if err != nil { + return err + } + return nil + }) + return err +} diff --git a/auth/jwt.go b/auth/jwt.go new file mode 100644 index 0000000..8ae4601 --- /dev/null +++ b/auth/jwt.go @@ -0,0 +1,193 @@ +package auth + +import ( + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt" + "github.com/rs/xid" + "github.com/rs/zerolog/log" + "github.com/spf13/viper" + "kumoly.io/kumoly/app/errors" + "kumoly.io/kumoly/app/system" +) + +const GinClaimKey = "claim" + +type Claims struct { + Uid string `json:"uid,omitempty"` + Groups []string `json:"grp,omitempty"` + Endpoint string `json:"ept,omitempty"` + IP string `json:"ip,omitempty"` + jwt.StandardClaims +} + +type Auth struct { + system.BaseService + CookieMode bool + CookieSecure bool + CookieSameSite http.SameSite + TokenExpire int64 + Endpoint string + AutoRenew bool + Secret string +} + +func NewAuth() *Auth { + return &Auth{ + CookieMode: true, + CookieSecure: strings.HasPrefix(viper.GetString("server.url"), "https"), + CookieSameSite: http.SameSiteLaxMode, + TokenExpire: viper.GetInt64("auth.expire"), + AutoRenew: true, + Secret: viper.GetString("auth.secret"), + } +} + +// Parse tok str to token object +func (srv Auth) Parse(tok string) (token *jwt.Token, err error) { + token, err = jwt.ParseWithClaims(tok, &Claims{}, func(token *jwt.Token) (interface{}, error) { + return []byte(srv.Secret), nil + }) + return +} + +// ParseClaims parse token string to claims object +func (srv Auth) ParseClaims(tok string) (claims *Claims, err error) { + token, err := srv.Parse(tok) + if err != nil { + return nil, err + } + claims, ok := token.Claims.(*Claims) + if !ok { + err = errors.New(http.StatusBadRequest, "ErrorUnknownClaims") + } + return +} + +// SetToken in header and cookie(if CookieMode) +func (srv Auth) SetToken(c *gin.Context, tok string) { + c.Header("Authorization", "Bearer "+tok) + if srv.CookieMode { + http.SetCookie(c.Writer, &http.Cookie{ + Name: viper.GetString("name") + "_bearer", + MaxAge: int(srv.TokenExpire), + Value: tok, + SameSite: srv.CookieSameSite, + Secure: srv.CookieSecure, + Path: "/", + }) + } +} + +// SetClaims directly to response +func (srv Auth) SetClaims(c *gin.Context, claims *Claims) error { + tok, err := srv.NewToken(*claims) + if err != nil { + return err + } + srv.SetToken(c, tok) + return nil +} + +// GetToken from header or cookie +func (srv Auth) GetToken(c *gin.Context) (tok string, err error) { + tok = strings.TrimPrefix(c.Request.Header.Get("Authorization"), "Bearer ") + if tok == "" && srv.CookieMode { + tok, err = c.Cookie(viper.GetString("name") + "_bearer") + } + if err != nil { + err = nil + return + } + if tok == "" { + err = errors.New(401, "ErrorTokenNotFound") + return + } + return tok, nil +} + +// GetClaims directly from http request +func (srv Auth) GetClaims(c *gin.Context) (claims *Claims, err error) { + tok, err := srv.GetToken(c) + if err != nil { + return + } + claims, err = srv.ParseClaims(tok) + return +} + +func (srv Auth) ClearToken(c *gin.Context) { + c.Writer.Header().Del("Authorization") + if srv.CookieMode { + http.SetCookie(c.Writer, &http.Cookie{ + Name: viper.GetString("name") + "_bearer", + MaxAge: -1, + Value: "", + SameSite: srv.CookieSameSite, + Secure: srv.CookieSecure, + Path: "/", + }) + } +} + +// New tok str +func (srv Auth) NewToken(claims Claims) (tok string, err error) { + if srv.TokenExpire > 0 && claims.ExpiresAt == 0 { + claims.ExpiresAt = time.Now().Unix() + srv.TokenExpire + } else if claims.ExpiresAt < 0 { + claims.ExpiresAt = 0 + } + claims.Issuer = viper.GetString("name") + claims.Id = xid.New().String() + claims.Endpoint = viper.GetString("domain") + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + tok, err = token.SignedString([]byte(srv.Secret)) + if err != nil { + log.Error().Str("mod", "auth").Err(err).Msg("NewToken") + } + return +} + +func (srv Auth) Middleware(c *gin.Context) { + claims, err := srv.GetClaims(c) + if err == nil { + c.Set(GinClaimKey, claims) + if srv.AutoRenew && claims.ExpiresAt != 0 { + claims.ExpiresAt = time.Now().Unix() + srv.TokenExpire + tok, err := srv.NewToken(*claims) + if err != nil { + log.Error().Str("mod", "auth").Err(err).Msg("Middleware") + } else { + srv.SetToken(c, tok) + } + } + } else { + log.Trace().Err(err).Msg("") + } +} + +func (srv Auth) Injector(router *gin.RouterGroup) *system.Inject { + return &system.Inject{ + Name: "auth.Auth", + InitFunc: func() error { + router.Use(srv.Middleware) + return nil + }, + } +} + +func GetContextClaims(c *gin.Context) (claims *Claims, err error) { + cl, ok := c.Get(GinClaimKey) + if !ok { + err = ErrorTokenNotValid + return + } + claims, ok = cl.(*Claims) + if !ok { + err = ErrorUnknownClaims + } + return +} diff --git a/auth/service.go b/auth/service.go new file mode 100644 index 0000000..85e3749 --- /dev/null +++ b/auth/service.go @@ -0,0 +1,81 @@ +package auth + +import ( + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/viper" + "gorm.io/gorm" + "kumoly.io/kumoly/app/server" + "kumoly.io/kumoly/app/store" + "kumoly.io/kumoly/app/system" +) + +var DB *gorm.DB + +func SetDB(db *gorm.DB) { + DB = db +} + +type Service struct { + system.BaseService + Logger zerolog.Logger + + server *server.Service +} + +func New(s *server.Service) *Service { + return &Service{server: s} +} + +func (srv Service) GetName() string { return "auth.Service" } +func (srv Service) GetDependencies() []string { return []string{"server.Service", "auth.Auth"} } + +func (srv Service) Init() error { + srv.Logger = log.With().Str("mod", "auth").Str("service", "auth.Service").Logger() + + srv.Logger.Debug().Msg("Migrating database for auth.Service ...") + if err := store.Migrate(&User{}, &Profile{}, &Group{}); err != nil { + srv.Logger.Error().Err(err).Msg("Migrating database") + return err + } + + srv.Logger.Debug().Msg("Checking db state") + + // add default group + if err := srv.SetDefaultGroups(); err != nil { + return err + } + + // add default admin account + if err := srv.SetDefaultAdmin( + viper.GetString("admin.user"), + viper.GetString("admin.passwd"), + ); err != nil { + return err + } + + return nil +} + +func (srv Service) Load() error { + srv.server.API.POST("/login", ApiLogin) + srv.server.API.POST("/logout", ApiLogout) + srv.server.API.GET("/whoami", ApiMe) + usrAPI := srv.server.API.Group("usr") + usrAPI.POST("/", ACAdmin(), ApiUserNew) + usrAPI.GET("/", ACAdmin(), ApiUsers) + usrAPI.DELETE("/:id", ACAdmin(), ApiUserDelete) + usrAPI.PUT("/passwd", ApiUserChangePasswd) + usrAPI.PUT("/activate/:id", ACAdmin(), ApiUserActivate) + usrAPI.PUT("/deactivate/:id", ACAdmin(), ApiUserDeactivate) + + grpAPI := srv.server.API.Group("grp") + grpAPI.GET("/", ACAdmin(), ApiGrps) + grpAPI.POST("/", ACAdmin(), ApiGrpNew) + grpAPI.PUT("/", ApiGrpUpdate) + grpAPI.DELETE("/:id", ACAdmin(), ApiGrpDel) + grpAPI.POST("/add/:uid/:gid", ApiGrpAssign) + grpAPI.POST("/remove/:uid/:gid", ACAdmin(), ApiGrpRemove) + + return nil +} diff --git a/auth/user.go b/auth/user.go new file mode 100644 index 0000000..5dd729f --- /dev/null +++ b/auth/user.go @@ -0,0 +1,72 @@ +package auth + +import ( + "time" + + "github.com/rs/xid" + "github.com/rs/zerolog/log" + "golang.org/x/crypto/bcrypt" + "gorm.io/gorm" +) + +//User User model +type User struct { + ID string `gorm:"primaryKey"` + CreatedAt time.Time + UpdatedAt time.Time + + Username string `gorm:"unique;not null"` + Password string `json:"-"` + + SSOEnabled bool + SSOTok string + Activated bool + LastLogin time.Time + LastLoginIP string + LoginFailed int + + Groups []*Group `gorm:"many2many:user_groups;"` + Profile Profile +} + +// Profile user extended information +type Profile struct { + ID uint `gorm:"primaryKey"` + CreatedAt time.Time + UpdatedAt time.Time + UserID string + + DisplayName string + Email string +} + +// BeforeCreate set UID +func (u *User) BeforeCreate(tx *gorm.DB) (err error) { + if u.ID == "" { + u.ID = xid.New().String() + } + u.Activated = true + u.LastLogin = time.Now() + u.LoginFailed = 0 + return +} + +// ChangePassword Change user password to *to*, return nil if success. +func (usr *User) ChangePassword(db *gorm.DB, to string) (err error) { + bytes, err := bcrypt.GenerateFromPassword([]byte(to), 14) + if err != nil { + return err + } + err = db.Transaction(func(tx *gorm.DB) error { + return tx.Model(usr).Update("password", string(bytes)).Error + }) + if err != nil { + log.Error().Str("mod", "auth").Err(err).Msg("ChangePassword") + } + return err +} + +// ValidatePassword validates user pass word ,return nil if correct. +func (usr *User) ValidatePassword(pwd string) error { + return bcrypt.CompareHashAndPassword([]byte(usr.Password), []byte(pwd)) +} diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..e69de29 diff --git a/errors/errors.go b/errors/errors.go new file mode 100644 index 0000000..8044dba --- /dev/null +++ b/errors/errors.go @@ -0,0 +1,39 @@ +package errors + +import ( + "encoding/json" + "fmt" +) + +type Error struct { + Code int `json:"code"` + ID string `json:"id"` + Message string `json:"msg"` + Tmpl string `json:"-"` +} + +func (e Error) New(v ...interface{}) Error { + e.Message = fmt.Sprintf(e.Tmpl, v...) + return e +} + +func (e Error) Error() string { + return e.Message +} + +func (e Error) String() string { + return e.Message +} + +func (e Error) Json() []byte { + data, _ := json.Marshal(e) + return data +} + +func New(code int, err string) *Error { + return &Error{Code: code, Message: err} +} + +func NewError(code int, err error) *Error { + return &Error{Code: code, Message: err.Error()} +} diff --git a/errors/net_errors.go b/errors/net_errors.go new file mode 100644 index 0000000..d9df3ba --- /dev/null +++ b/errors/net_errors.go @@ -0,0 +1,27 @@ +package errors + +import "net/http" + +var ErrorNotFound = Error{ + Code: http.StatusNotFound, + ID: "ErrorNotFound", + Message: "not found", +} + +var ErrorUnauthorized = Error{ + Code: http.StatusUnauthorized, + ID: "ErrorUnauthorized", + Message: "unauthorized", +} + +var ErrorForbidden = Error{ + Code: http.StatusForbidden, + ID: "ErrorForbidden", + Message: "permission denied", +} + +var ErrorBadRequest = Error{ + Code: http.StatusBadRequest, + ID: "ErrorBadRequest", + Message: "bad request", +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6e996c8 --- /dev/null +++ b/go.mod @@ -0,0 +1,58 @@ +module kumoly.io/kumoly/app + +go 1.17 + +require ( + github.com/gin-gonic/gin v1.7.7 + github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/robfig/cron/v3 v3.0.1 + github.com/rs/xid v1.3.0 + github.com/rs/zerolog v1.26.1 + github.com/spf13/viper v1.10.1 + golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e + gorm.io/driver/mysql v1.2.1 + gorm.io/driver/postgres v1.2.3 + gorm.io/driver/sqlite v1.2.6 + gorm.io/gorm v1.22.4 +) + +require ( + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.13.0 // indirect + github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/validator/v10 v10.4.1 // indirect + github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.10.1 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.2.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.9.0 // indirect + github.com/jackc/pgx/v4 v4.14.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.3 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/leodido/go-urn v1.2.0 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-sqlite3 v1.14.9 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml v1.9.4 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/ugorji/go/codec v1.1.7 // indirect + golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect + golang.org/x/text v0.3.7 // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..38b75e8 --- /dev/null +++ b/go.sum @@ -0,0 +1,292 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= +github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.10.1 h1:DzdIHIjG1AxGwoEEqS+mGsURyjt4enSmqzACXvVzOT8= +github.com/jackc/pgconn v1.10.1/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.9.0 h1:/SH1RxEtltvJgsDqp3TbiTFApD3mey3iygpuEGeuBXk= +github.com/jackc/pgtype v1.9.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.14.0 h1:TgdrmgnM7VY72EuSQzBbBd4JA1RLqJolrw9nQVZABVc= +github.com/jackc/pgx/v4 v4.14.0/go.mod h1:jT3ibf/A0ZVCp89rtCIN0zCJxcE74ypROmHEZYsG/j8= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.3 h1:PlHq1bSCSZL9K0wUhbm2pGLoTWs2GwVhsP6emvGV/ZI= +github.com/jinzhu/now v1.1.3/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= +github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= +github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= +github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= +github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e h1:1SzTfNOXwIS2oWiMF+6qu0OUDKb0dauo6MoDUQyu+yU= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gorm.io/driver/mysql v1.2.1 h1:h+3f1l9Ng2C072Y2tIiLgPpWN78r1KXL7bHJ0nTjlhU= +gorm.io/driver/mysql v1.2.1/go.mod h1:qsiz+XcAyMrS6QY+X3M9R6b/lKM1imKmcuK9kac5LTo= +gorm.io/driver/postgres v1.2.3 h1:f4t0TmNMy9gh3TU2PX+EppoA6YsgFnyq8Ojtddb42To= +gorm.io/driver/postgres v1.2.3/go.mod h1:pJV6RgYQPG47aM1f0QeOzFH9HxQc8JcmAgjRCgS0wjs= +gorm.io/driver/sqlite v1.2.6 h1:SStaH/b+280M7C8vXeZLz/zo9cLQmIGwwj3cSj7p6l4= +gorm.io/driver/sqlite v1.2.6/go.mod h1:gyoX0vHiiwi0g49tv+x2E7l8ksauLK0U/gShcdUsjWY= +gorm.io/gorm v1.22.3/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.22.4 h1:8aPcyEJhY0MAt8aY6Dc524Pn+pO29K+ydu+e/cXSpQM= +gorm.io/gorm v1.22.4/go.mod h1:1aeVC+pe9ZmvKZban/gW4QPra7PRoTEssyc922qCAkk= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/main.go b/main.go new file mode 100644 index 0000000..674cc2f --- /dev/null +++ b/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "kumoly.io/kumoly/app/auth" + "kumoly.io/kumoly/app/server" + "kumoly.io/kumoly/app/store" + "kumoly.io/kumoly/app/system" + "kumoly.io/kumoly/app/task" +) + +func main() { + store.Setup() + sys := system.New() + server := server.New("app") + + auth.Setup() + auth.SetDB(store.DB) + + sys.Inject(auth.Injector(server.API)) + sys.Append(server, auth.New(server), &task.Service{}) + sys.Start() + +} diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100644 index 0000000..e69de29 diff --git a/server/helper.go b/server/helper.go new file mode 100644 index 0000000..286e501 --- /dev/null +++ b/server/helper.go @@ -0,0 +1,47 @@ +package server + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/rs/xid" +) + +type Response struct { + Status int `json:"status"` + ID string `json:"id"` + Code string `json:"code,omitempty"` + Data interface{} `json:"data,omitempty"` +} + +func Res(c *gin.Context, res *Response) { + res.ID = xid.New().String() + if res.Status == 0 { + res.Status = 200 + } + c.JSON(res.Status, res) +} + +func OK(c *gin.Context, data interface{}) { + c.JSON(http.StatusOK, Response{ + Status: http.StatusOK, + ID: xid.New().String(), + Data: data, + }) +} + +func Error(c *gin.Context, data interface{}) { + c.JSON(http.StatusInternalServerError, Response{ + Status: http.StatusInternalServerError, + ID: xid.New().String(), + Data: data, + }) +} + +func WithCode(c *gin.Context, code int, data interface{}) { + c.JSON(code, Response{ + Status: code, + ID: xid.New().String(), + Data: data, + }) +} diff --git a/server/middleware.go b/server/middleware.go new file mode 100644 index 0000000..e8de7bc --- /dev/null +++ b/server/middleware.go @@ -0,0 +1,56 @@ +package server + +import ( + "fmt" + "net" + "time" + + "github.com/gin-gonic/gin" + "github.com/rs/zerolog" + "kumoly.io/kumoly/app/errors" +) + +func (srv *Service) Default(c *gin.Context) { + path := c.Request.URL.Path + start := time.Now() + defer func() { + var cl *zerolog.Event + err := recover() + if err != nil { + cl = srv.l.Error() + switch v := err.(type) { + case errors.Error: + c.AbortWithStatusJSON(v.Code, v) + cl.Err(v) + case error: + c.String(500, v.Error()) + c.Abort() + cl.Err(v) + default: + c.String(500, fmt.Sprint(err)) + c.Abort() + cl.Str("error", fmt.Sprint(err)) + } + } else if c.Writer.Status() >= 500 { + cl = srv.l.Error().Strs("error", c.Errors.Errors()) + } else { + cl = srv.l.Info() + } + if srv.SkipLog != nil && srv.SkipLog(c) { + return + } + cl. + Str("method", c.Request.Method). + Str("ip", c.ClientIP()). + Int("status", c.Writer.Status()). + Dur("duration", time.Since(start)). + Str("url", path). + Msg("") + }() + if srv.Allow != nil { + if !srv.Allow.Contains(net.ParseIP(c.ClientIP())) { + panic(errors.ErrorForbidden) + } + } + c.Next() +} diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..0fb4669 --- /dev/null +++ b/server/server.go @@ -0,0 +1,12 @@ +package server + +import "github.com/spf13/viper" + +func init() { + // Server + viper.SetDefault("server.port", "80") + viper.SetDefault("server.host", "0.0.0.0") + viper.SetDefault("server.url", "http://localhost") + viper.SetDefault("server.allow", "") + +} diff --git a/server/service.go b/server/service.go new file mode 100644 index 0000000..a3d51a9 --- /dev/null +++ b/server/service.go @@ -0,0 +1,78 @@ +package server + +import ( + "fmt" + "net" + + "github.com/gin-gonic/gin" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/viper" +) + +type SkipLogFunc func(c *gin.Context) bool + +type Service struct { + Name string + Server *gin.Engine + API *gin.RouterGroup + SkipLog SkipLogFunc + Addr string + Allow *net.IPNet + + l zerolog.Logger +} + +func New(name string) *Service { + if viper.GetBool("prod") { + gin.SetMode(gin.ReleaseMode) + } + srv := &Service{ + Name: name, + Server: gin.New(), + Addr: fmt.Sprintf("%s:%s", viper.GetString("server.host"), viper.GetString("server.port")), + } + srv.Server.Use(srv.Default) + srv.API = srv.Server.Group("/api") + if ipnetstr := viper.GetString("server.allow"); ipnetstr != "" { + _, ipnet, err := net.ParseCIDR(ipnetstr) + if err != nil { + log.Panic().Str("service", "server.Service").Str("name", name).Err(err).Msg("ParseCIDR error") + } + srv.Allow = ipnet + } + return srv +} +func (srv *Service) Init() error { + srv.l = log.With().Str("service", "server.Service").Str("name", srv.Name).Logger() + return nil +} +func (srv *Service) Load() error { + return nil +} +func (srv *Service) Main() error { + srv.l.Info().Msgf("Server started on %s", srv.Addr) + go func() { + err := srv.Server.Run(srv.Addr) + if err != nil { + srv.l.Panic().Err(err).Msg("Server.Run error") + } + }() + return nil +} +func (srv *Service) Del() { + +} +func (srv Service) Health() error { + return nil +} + +func (srv Service) GetName() string { + return "server.Service" +} +func (srv Service) GetDependencies() []string { + return nil +} +func (srv Service) IsService() bool { + return true +} diff --git a/store/db.go b/store/db.go new file mode 100644 index 0000000..31cb054 --- /dev/null +++ b/store/db.go @@ -0,0 +1,194 @@ +package store + +import ( + "fmt" + "path/filepath" + "time" + + "github.com/rs/zerolog/log" + "github.com/spf13/viper" + "gorm.io/driver/mysql" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" + "kumoly.io/kumoly/app/util" +) + +type DBTYPE string + +const ( + MYSQL DBTYPE = "mysql" + SQLITE DBTYPE = "sqlite" + POSTGRES DBTYPE = "postgres" +) + +type Store struct { + TYPE DBTYPE + DB *gorm.DB + User string + Password string + Host string + Port string + Name string + AutoMigrate bool + Prod bool + Path string + + config *gorm.Config +} + +var DB *gorm.DB +var std *Store + +func init() { + // Database + // type [mysql,sqlite] + viper.SetDefault("db.type", string(MYSQL)) + + // mysql default + viper.SetDefault("db.user", "root") + viper.SetDefault("db.passwd", "admin") + viper.SetDefault("db.host", "127.0.0.1") + viper.SetDefault("db.port", "3306") + viper.SetDefault("db.name", "app") + + viper.SetDefault("db.reconnect_interval", 5) + viper.SetDefault("db.automigrate", true) +} + +// Init initialize default db using New() followed by Connect() +func Setup() { + var err error + std = New(DBTYPE(viper.GetString("db.type"))) + DB, err = std.Connect() + if err != nil { + log.Error().Str("mod", "store").Err(err).Msg("std connection error") + panic(err) + } +} + +func Migrate(dst ...interface{}) error { + return std.Migrate(dst...) +} + +func New(t DBTYPE) *Store { + s := &Store{ + TYPE: t, + Prod: viper.GetBool("prod"), + Name: viper.GetString("db.name"), + AutoMigrate: viper.GetBool("db.automigrate"), + } + if t == MYSQL || t == POSTGRES { + s.User = viper.GetString("db.user") + s.Password = viper.GetString("db.passwd") + s.Host = viper.GetString("db.host") + s.Port = viper.GetString("db.port") + } else if t == SQLITE { + s.Path = viper.GetString("data") + } else { + err := fmt.Errorf("unknown db type %s", t) + log.Error().Str("mod", "store").Err(err).Msg("unknown db type") + panic(err) + } + return s +} + +// Connect to db and store db connection to var DB +func (s *Store) Connect() (db *gorm.DB, err error) { + s.config = &gorm.Config{ + Logger: logger.Default, + SkipDefaultTransaction: true, + } + if s.Prod { + s.config.Logger = logger.Discard + } + switch s.TYPE { + case MYSQL: + return s.DB, s.mysqlConnector() + case POSTGRES: + return s.DB, s.postgresConnector() + case SQLITE: + return s.DB, s.sqliteConnector() + default: + return nil, fmt.Errorf("unknown db type %s", s.TYPE) + } +} + +func (s *Store) postgresConnector() error { + log.Info().Str("mod", "store").Msg("Connecting to postgres...") + dsn := fmt.Sprintf( + "host=%v user=%v password=%v dbname=%v port=%v sslmode=disable TimeZone=Asia/Taipei", + s.Host, s.User, s.Password, s.Name, s.Port, + ) + for { + db, err := gorm.Open(postgres.New(postgres.Config{ + DSN: dsn, // data source name + // PreferSimpleProtocol: true, // disables implicit prepared statement usage + }), s.config) + if err == nil { + s.DB = db + break + } + inter := viper.GetDuration("db.reconnect_interval") + if inter <= 0 { + inter = 5 + } + log.Warn().Str("mod", "store").Err(err).Msg("Unable to connect to database") + log.Warn().Str("mod", "store").Msgf("Retrying in %v second.", inter) + time.Sleep(time.Second * inter) + } + log.Info().Str("mod", "store").Msg("Connection to postgres, ok.") + return nil +} + +//mysqlConnector connection +func (s *Store) mysqlConnector() error { + log.Info().Str("mod", "store").Msg("Connecting to mysql...") + dsn := fmt.Sprintf( + "%v:%v@tcp(%v:%v)/%v?charset=utf8mb4&parseTime=True&loc=Local", + s.User, s.Password, s.Host, s.Port, s.Name, + ) + // log.Debug(dsn) + for { + db, err := gorm.Open(mysql.New(mysql.Config{ + DSN: dsn, // data source name + DefaultStringSize: 256, // default size for string fields + }), s.config) + if err == nil { + s.DB = db + break + } + inter := viper.GetDuration("db.reconnect_interval") + if inter <= 0 { + inter = 5 + } + log.Warn().Str("mod", "store").Err(err).Msg("Unable to connect to database") + log.Warn().Str("mod", "store").Msgf("Retrying in %v second.", inter) + time.Sleep(time.Second * inter) + } + log.Info().Str("mod", "store").Msg("Connection to mysql, ok.") + return nil +} + +//sqliteConnector connection +func (s *Store) sqliteConnector() error { + util.Mkdir(s.Path) + dbPath := filepath.Join(s.Path, s.Name+".db") + log.Info().Str("mod", "store").Str("path", dbPath).Msg("Connecting to sqlite...") + db, err := gorm.Open(sqlite.Open(dbPath), s.config) + if err != nil { + log.Error().Str("mod", "store").Err(err).Msg("failed to connect database") + return err + } + s.DB = db + return nil +} + +func (s *Store) Migrate(dst ...interface{}) error { + if !s.AutoMigrate { + log.Debug().Str("mod", "store").Msg("AutoMigration is set to off, migration skipped") + return nil + } + return s.DB.AutoMigrate(dst...) +} diff --git a/system/helper.go b/system/helper.go new file mode 100644 index 0000000..243843a --- /dev/null +++ b/system/helper.go @@ -0,0 +1,145 @@ +package system + +import ( + "fmt" + + "kumoly.io/kumoly/app/util" +) + +type BaseService struct{} + +func (srv BaseService) GetName() string { return "base" } +func (srv BaseService) GetDependencies() []string { return []string{""} } +func (srv BaseService) IsService() bool { return false } +func (srv BaseService) Init() error { return nil } +func (srv BaseService) Load() error { return nil } +func (srv BaseService) Main() error { return nil } +func (srv BaseService) Del() {} +func (srv BaseService) Health() error { return nil } + +type EasyBaseService struct { + Name string + Dependencies []string + IsServ bool +} + +func (srv EasyBaseService) GetName() string { return srv.Name } +func (srv EasyBaseService) GetDependencies() []string { return srv.Dependencies } +func (srv EasyBaseService) IsService() bool { return srv.IsServ } +func (srv EasyBaseService) Init() error { return nil } +func (srv EasyBaseService) Load() error { return nil } +func (srv EasyBaseService) Main() error { return nil } +func (srv EasyBaseService) Del() {} +func (srv EasyBaseService) Health() error { return nil } + +type Inject struct { + Name string + Dependencies []string + IsServ bool + + InitFunc func() error + LoadFunc func() error + MainFunc func() error + DelFunc func() + HealthFunc func() error +} + +func (srv Inject) GetName() string { return srv.Name } +func (srv Inject) GetDependencies() []string { return srv.Dependencies } +func (srv Inject) IsService() bool { return srv.IsServ } +func (srv Inject) Init() error { + if srv.InitFunc == nil { + return nil + } + return srv.InitFunc() +} +func (srv Inject) Load() error { + if srv.LoadFunc == nil { + return nil + } + return srv.LoadFunc() +} +func (srv Inject) Main() error { + if srv.MainFunc == nil { + return nil + } + return srv.MainFunc() +} +func (srv Inject) Del() { + if srv.HealthFunc == nil { + return + } + srv.DelFunc() +} +func (srv Inject) Health() error { + if srv.HealthFunc == nil { + return nil + } + return srv.HealthFunc() +} + +func removeNode(s []*node, i int) []*node { + s[i] = s[len(s)-1] + return s[:len(s)-1] +} + +func removeEdge(s []string, i int) []string { + s[i] = s[len(s)-1] + return s[:len(s)-1] +} + +type node struct { + srv *Service + name string + edge []string +} + +func topo(nodes []*node) []*node { + // check + names := map[string]bool{} + for _, v := range nodes { + names[v.name] = true + } + for _, n := range nodes { + for _, e := range n.edge { + if !names[e] { + err := fmt.Errorf("dependency not found: %v", e) + l.Error().Err(err).Msg("") + panic(err) + } + } + } + + // start + ret := []*node{} + for i := 0; i <= len(nodes); i++ { + if len(nodes) == 0 { + break + } + if i == len(nodes) { + i = 0 + } + if nodes[i].edge == nil || len(nodes[i].edge) == 0 { + n := nodes[i].name + ret = append(ret, nodes[i]) + nodes = removeNode(nodes, i) + for j := 0; j < len(nodes); j++ { + for k := 0; k < len(nodes[j].edge); k++ { + if nodes[j].edge[k] == n { + nodes[j].edge = removeEdge(nodes[j].edge, k) + } + } + } + i-- + } + } + return ret +} + +// Inject a system.Inject and uses caller as name +func (sys *System) Inject(i *Inject) { + if i.Name == "" { + i.Name = util.Caller(2) + } + sys.Append(i) +} diff --git a/system/helper_test.go b/system/helper_test.go new file mode 100644 index 0000000..498408b --- /dev/null +++ b/system/helper_test.go @@ -0,0 +1,89 @@ +package system + +import ( + "fmt" + "testing" + "time" + + "github.com/spf13/viper" +) + +func TestCoreInject(t *testing.T) { + sys := New() + viper.Set("log.level", 15) + b1 := EasyBaseService{"b1", []string{}, true} + b2 := EasyBaseService{"b2", []string{"b1"}, true} + b3 := EasyBaseService{"b3", []string{"b1"}, true} + b4 := EasyBaseService{"b4", []string{"b3", "b1"}, true} + sys.Append(b1, b2, b3, b4) + sys.Inject(&Inject{ + Dependencies: []string{"b2"}, + MainFunc: func() error { l.Info().Msg("testing, injection"); return nil }, + }) + go sys.Start() + time.Sleep(time.Second * 1) + sys.Stop() + <-sys.Done() +} + +func TestTopo(t *testing.T) { + nodes := []*node{} + nodes = append(nodes, + &node{ + name: "n1", + edge: []string{"n2"}, + }, + &node{ + name: "n2", + }, + &node{ + name: "n3", + edge: []string{"n1", "n2"}, + }, + &node{ + name: "n4", + edge: []string{"n3"}, + }, + &node{ + name: "n5", + }, + &node{ + name: "n6", + edge: []string{"n4", "n5"}, + }, + ) + + result := topo(nodes) + for i := range result { + fmt.Print(result[i].name, ", ") + } + fmt.Print("\n") + +} + +func TestTopoError(t *testing.T) { + nodes := []*node{} + nodes = append(nodes, + &node{ + name: "n1", + edge: []string{"n2"}, + }, + &node{ + name: "n2", + edge: []string{"n3"}, + }, + ) + + defer func() { + if r := recover(); r == nil { + t.Errorf("The code did not panic") + } + }() + + result := topo(nodes) + for i := range result { + fmt.Print(result[i].name, ", ") + } + fmt.Print("\n") + +} diff --git a/system/profile.go b/system/profile.go new file mode 100644 index 0000000..0349d31 --- /dev/null +++ b/system/profile.go @@ -0,0 +1,38 @@ +package system + +import ( + "runtime" + + "github.com/spf13/viper" +) + +type Profile struct { + App string + Domain string + LogLevel int + Prod bool + Alloc uint64 + TotalAlloc uint64 + SysMem uint64 + NumGC uint32 +} + +func GetProfile() *Profile { + var m runtime.MemStats + runtime.ReadMemStats(&m) + + return &Profile{ + App: viper.GetString("name"), + Domain: viper.GetString("domain"), + Prod: viper.GetBool("prod"), + LogLevel: viper.GetInt("log.level"), + Alloc: bToMb(m.Alloc), + TotalAlloc: bToMb(m.TotalAlloc), + SysMem: bToMb(m.Sys), + NumGC: m.NumGC, + } +} + +func bToMb(b uint64) uint64 { + return b / 1024 / 1024 +} diff --git a/system/profile_test.go b/system/profile_test.go new file mode 100644 index 0000000..0ced003 --- /dev/null +++ b/system/profile_test.go @@ -0,0 +1,33 @@ +package system + +import ( + "fmt" + "runtime" + "testing" + "time" +) + +func TestProfile(t *testing.T) { + fmt.Printf("%+v\n", GetProfile()) + var overall [][]int + for i := 0; i < 4; i++ { + + // Allocate memory using make() and append to overall (so it doesn't get + // garbage collected). This is to create an ever increasing memory usage + // which we can track. We're just using []int as an example. + a := make([]int, 0, 999999) + overall = append(overall, a) + + // Print our memory usage at each interval + + fmt.Printf("%+v\n", GetProfile()) + time.Sleep(time.Second) + } + + // Clear our memory and print usage, unless the GC has run 'Alloc' will remain the same + overall = nil + fmt.Printf("%+v\n", GetProfile()) + runtime.GC() + fmt.Printf("%+v\n", GetProfile()) + +} diff --git a/system/setup.go b/system/setup.go new file mode 100644 index 0000000..753f44e --- /dev/null +++ b/system/setup.go @@ -0,0 +1,127 @@ +package system + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/viper" + "kumoly.io/kumoly/app/util" +) + +func init() { + Home, err := os.UserHomeDir() + if err != nil { + Home = "/" + } + + // Setup defaults + // ========================================================= + // App + viper.SetDefault("name", "app") + viper.SetDefault("domain", "kumoly.io") + viper.SetDefault("timezone", "Asia/Taipei") + // mode [prod, dev] + viper.SetDefault("prod", PROD) + + // log.level : [-1(trace):5(panic)] 7 to disable + viper.SetDefault("log.level", int(zerolog.InfoLevel)) + viper.SetDefault("log.pretty", false) + + // data: data dir + viper.SetDefault("data", filepath.Join(Home, ".kumoly")) + + // ========================================================= + // viper settings + viper.SetEnvPrefix("app") + replacer := strings.NewReplacer(".", "_") + viper.SetEnvKeyReplacer(replacer) + viper.AutomaticEnv() + viper.SetConfigType("json") +} + +func setup() { + zerolog.TimeFieldFormat = zerolog.TimeFormatUnix + zerolog.DurationFieldInteger = true + zerolog.SetGlobalLevel(zerolog.Level(viper.GetInt("log.level"))) + if !viper.GetBool("prod") { + log.Logger = log.With().Caller().Logger() + } + if viper.GetBool("log.pretty") { + log.Logger = log.Output(zerolog.ConsoleWriter{ + Out: os.Stdout, + TimeFormat: "2006/01/02 15:04:05", + FormatCaller: func(i interface{}) string { + var c string + if cc, ok := i.(string); ok { + c = cc + } + // if len(c) > 0 { + + // } + return c + }, + }) + } + l = log.With().Str("mod", "system").Logger() + + tz, err := time.LoadLocation(viper.GetString("timezone")) + if err == nil { + time.Local = tz + } else { + l.Error().Err(err).Msg("setup error") + } +} + +func SetConfigPath(path string) { + ConfigRead(path) +} + +func SetDataPath(path string) { + viper.Set("data", path) +} + +// ConfigRead read in config +func ConfigRead(cfgFile string) { + if cfgFile != "" { + viper.SetConfigFile(cfgFile) + } else { + // read config from default path + viper.SetConfigName("app") + viper.AddConfigPath(viper.GetString("data")) + } + if err := viper.ReadInConfig(); err != nil { + l.Error().Err(err).Msg("ReadInConfig error") + } + setup() +} + +// ConfigWrite write current config to file +func ConfigWrite(file string) { + viper.WriteConfigAs(file) +} + +// ConfigShow print config to stdout +func ConfigShow(format string) (string, error) { + if format == "" { + format = "json" + } + path := filepath.Join(viper.GetString("data"), "tmp") + util.Mkdir(path) + file := filepath.Join(path, "config."+format) + err := viper.WriteConfigAs(file) + if err != nil { + l.Error().Err(err).Msg("WriteConfigAs error") + return "", err + } + data, err := ioutil.ReadFile(file) + if err != nil { + l.Error().Err(err).Msg("ReadFile error") + return "", err + } + return string(data), nil +} diff --git a/system/system.go b/system/system.go new file mode 100644 index 0000000..01fea71 --- /dev/null +++ b/system/system.go @@ -0,0 +1,222 @@ +package system + +import ( + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/viper" +) + +var PROD = true +var l zerolog.Logger + +func init() { + l = log.With().Str("mod", "system").Logger() + viper.SetDefault("system.terminate_timeout", 10) +} + +type Service interface { + Init() error + Load() error + Main() error + Del() + Health() error + GetName() string + GetDependencies() []string + IsService() bool +} + +type state int + +const ( + sys_init = iota + sys_load + sys_main + sys_wait + sys_term + sys_exit + sys_restart +) + +type System struct { + services []Service + isService bool + status state + lock sync.Mutex + term chan struct{} + done chan struct{} + notifyDone chan struct{} + quit chan os.Signal +} + +func (sys *System) Append(srv ...Service) { + if sys.services == nil { + sys.services = make([]Service, 0) + } + sys.services = append(sys.services, srv...) +} + +func (sys *System) order() { + nodes := []*node{} + for i := range sys.services { + nodes = append(nodes, &node{ + srv: &sys.services[i], + name: sys.services[i].GetName(), + edge: sys.services[i].GetDependencies(), + }) + } + nodes = topo(nodes) + sys.services = make([]Service, 0) + for i := range nodes { + sys.services = append(sys.services, *nodes[i].srv) + } + if !PROD { + seq := "" + for i := range sys.services { + seq += sys.services[i].GetName() + if i < len(sys.services)-1 { + seq += ", " + } + } + l.Debug().Msg(seq) + } +} + +func New() *System { + sys := &System{} + sys.done = make(chan struct{}, 1) + sys.term = make(chan struct{}, 1) + sys.notifyDone = make(chan struct{}, 1) + sys.quit = make(chan os.Signal, 1) + signal.Notify(sys.quit, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + return sys +} + +func (sys *System) Start() { + setup() + sys.order() + sys.status = sys_init + go func() { + l.Info().Msg("Initiating...") + if err := sys.init(); err != nil { + sys.quit <- syscall.SIGTERM + return + } + sys.status = sys_load + + l.Info().Msg("Loading...") + if err := sys.load(); err != nil { + sys.quit <- syscall.SIGTERM + return + } + + l.Info().Msg("Starting...") + + sys.status = sys_main + sys.main() + if !sys.isService { + sys.status = sys_wait + sys.quit <- syscall.SIGTERM + } + }() + <-sys.quit + isRestart := false + if sys.status == sys_restart { + isRestart = true + } + + l.Info().Msg("Terminating...") + sys.status = sys_term + go func() { + time.Sleep(time.Second * time.Duration(viper.GetInt("system.terminate_timeout"))) + if sys.status == sys_term { + l.Warn().Msg("System Termination Timeout reached! Force Terminating.") + sys.done <- struct{}{} + } + }() + go func() { + sys.del() + sys.done <- struct{}{} + }() + + <-sys.done + sys.status = sys_exit + l.Info().Msg("Service terminated.") + if isRestart { + l.Info().Msg("Restarting...") + sys.term <- struct{}{} + } else { + sys.notifyDone <- struct{}{} + } +} + +// Stop stops the system +func (sys *System) Stop() { + sys.quit <- syscall.SIGTERM +} + +func (sys *System) Restart() { + sys.lock.Lock() + defer sys.lock.Unlock() + sys.status = sys_restart + sys.Stop() + <-sys.term + go sys.Start() +} + +// Done listens to system termination +func (sys *System) Done() chan struct{} { + return sys.notifyDone +} + +func (sys *System) init() error { + for i := range sys.services { + l.Debug().Int("step", i).Str("srvname", sys.services[i].GetName()).Msg("") + if err := sys.services[i].Init(); err != nil { + l.Error().Err(err).Int("step", i).Str("srvname", sys.services[i].GetName()).Msg("") + return err + } + } + return nil +} +func (sys *System) load() error { + for i := range sys.services { + l.Debug().Int("step", i).Str("srvname", sys.services[i].GetName()).Msg("") + if err := sys.services[i].Load(); err != nil { + l.Error().Err(err).Int("step", i).Str("srvname", sys.services[i].GetName()).Msg("") + return err + } + } + return nil +} +func (sys *System) main() error { + for i := range sys.services { + if sys.services[i].IsService() { + sys.isService = true + } + l.Debug().Int("step", i).Str("srvname", sys.services[i].GetName()).Msg("") + if err := sys.services[i].Main(); err != nil { + l.Error().Err(err).Int("step", i).Str("srvname", sys.services[i].GetName()).Msg("") + return err + } + } + return nil +} +func (sys *System) del() error { + var wg sync.WaitGroup + for i := range sys.services { + wg.Add(1) + go func(index int, srv Service) { + l.Debug().Int("step", index).Str("srvname", sys.services[index].GetName()).Msg("") + srv.Del() + wg.Done() + }(i, sys.services[i]) + } + wg.Wait() + return nil +} diff --git a/system/system_test.go b/system/system_test.go new file mode 100644 index 0000000..e33e931 --- /dev/null +++ b/system/system_test.go @@ -0,0 +1,24 @@ +package system + +import ( + "testing" + "time" +) + +func TestCoreStart(t *testing.T) { + sys := New() + + b1 := EasyBaseService{"b1", []string{}, true} + b2 := EasyBaseService{"b2", []string{"b1"}, true} + b3 := EasyBaseService{"b3", []string{"b1"}, true} + b4 := EasyBaseService{"b4", []string{"b3", "b6"}, true} + b5 := EasyBaseService{"b5", []string{"b4", "b1"}, true} + b6 := EasyBaseService{"b6", []string{"b1", "b2"}, true} + sys.Append(b1, b2, b3, b4, b5, b6) + + go sys.Start() + time.Sleep(time.Second * 1) + sys.Stop() + <-sys.Done() + l.Debug().Msg("test finish") +} diff --git a/task/logger.go b/task/logger.go new file mode 100644 index 0000000..90b492a --- /dev/null +++ b/task/logger.go @@ -0,0 +1,39 @@ +package task + +import ( + "fmt" + "time" + + "github.com/rs/zerolog" +) + +type mylogger struct { +} + +func (ml *mylogger) Info(msg string, keysAndValues ...interface{}) { + chain := l.Info() + formatKV(chain, keysAndValues...) + chain.Msg(msg) +} + +func (ml *mylogger) Error(err error, msg string, keysAndValues ...interface{}) { + chain := l.Error().Err(err) + formatKV(chain, keysAndValues...) + chain.Msg(msg) +} + +func formatKV(chain *zerolog.Event, keysAndValues ...interface{}) { + count := len(keysAndValues) + for i := 0; i < count; i += 2 { + switch v := keysAndValues[i+1].(type) { + case string: + chain.Str(keysAndValues[i].(string), v) + case int: + chain.Int(keysAndValues[i].(string), v) + case time.Time: + chain.Time(keysAndValues[i].(string), v) + default: + chain.Str(keysAndValues[i].(string), fmt.Sprint(v)) + } + } +} diff --git a/task/profile.go b/task/profile.go new file mode 100644 index 0000000..51bca41 --- /dev/null +++ b/task/profile.go @@ -0,0 +1,65 @@ +package task + +import ( + "time" + + cron "github.com/robfig/cron/v3" +) + +var Reporter chan *Report + +type Report struct { + // Entry + Next time.Time `json:"next"` + Prev time.Time `json:"prev"` + Entry cron.EntryID `json:"entry_id"` + + // Task + Name string `json:"name"` + ID string `json:"id"` + Description string `json:"description"` + Group string `json:"grp"` + + // _task + Spec string `json:"spec"` + Once bool `json:"once"` + Args []interface{} `json:"args"` + Carry interface{} `json:"carry"` + + // Report + Error error `json:"error,omitempty"` +} + +type Profile struct { + Name string `json:"name"` + Scheduled []*Report `json:"scheduled"` +} + +func GetProfile() *Profile { + p := &Profile{ + Name: "core/task", + } + entries := c.Entries() + for i := 0; i < len(entries); i++ { + t, ok := entries[i].Job.(*_task) + if !ok { + continue + } + r := &Report{ + Next: entries[i].Next, + Prev: entries[i].Prev, + Entry: entries[i].ID, + Name: t.Task.Name, + ID: t.Task.ID, + Description: t.Task.Description, + Group: t.Task.Group, + Spec: t.spec, + Once: t.once, + Args: t.args, + Carry: t.carry, + } + p.Scheduled = append(p.Scheduled, r) + } + + return p +} diff --git a/task/service.go b/task/service.go new file mode 100644 index 0000000..4e9aafa --- /dev/null +++ b/task/service.go @@ -0,0 +1,23 @@ +package task + +type Service struct{} + +func (srv Service) GetName() string { return "task.Service" } +func (srv Service) GetDependencies() []string { return []string{} } +func (srv Service) IsService() bool { return true } +func (srv Service) Init() error { + Init() + return nil +} +func (srv Service) Load() error { return nil } +func (srv Service) Main() error { + Start() + return nil +} +func (srv Service) Del() { + if c == nil { + return + } + <-Stop().Done() +} +func (srv Service) Health() error { return nil } diff --git a/task/store.go b/task/store.go new file mode 100644 index 0000000..2487d4f --- /dev/null +++ b/task/store.go @@ -0,0 +1,108 @@ +package task + +import ( + "fmt" + "sync" + "time" + + cron "github.com/robfig/cron/v3" + "github.com/rs/xid" + "kumoly.io/kumoly/app/util" +) + +var stlok sync.Mutex + +var tasks = map[string]*Task{} + +func AddTask(t *Task) { + stlok.Lock() + defer stlok.Unlock() + if t.ID == "" { + l.Warn().Str("taskname", t.Name).Msg("no ID, using random id") + t.ID = xid.New().String() + } + _, ok := tasks[t.ID] + if ok { + l.Warn().Str("taskname", t.Name).Str("taskid", t.ID).Msg("task is already in store, skipping") + } else { + tasks[t.ID] = t + } +} + +func GetTasks() map[string]*Task { + return tasks +} + +func GetTask(id string) *Task { + return tasks[id] +} + +func NewTask() *Task { + return &Task{ + Name: util.Caller(2), + ID: xid.New().String(), + } +} + +func RunFunc(spec string, cmd func()) error { + th := &Task{ + Name: util.Caller(2), + ID: xid.New().String(), + Func: func(carry *interface{}, i ...interface{}) error { cmd(); return nil }, + } + t := &_task{Task: th, spec: spec} + id, err := c.AddJob(spec, t) + t.id = id + return err +} + +func AddAndRunTask(spec string, t *Task, args ...interface{}) error { + AddTask(t) + _t := &_task{Task: t, args: args, spec: spec} + id, err := c.AddJob(spec, _t) + _t.id = id + return err +} + +func RunTask(spec, id string, args ...interface{}) error { + t := GetTask(id) + if t == nil { + return fmt.Errorf("no task with id") + } + _t := &_task{Task: t, args: args, spec: spec} + _id, err := c.AddJob(spec, _t) + _t.id = _id + return err +} + +func RunTaskAt(at time.Time, id string, args ...interface{}) error { + next := time.Until(at).Round(time.Second) + if next < 0 { + return fmt.Errorf("%v is passed by %s", at, next) + } + t := GetTask(id) + if t == nil { + return fmt.Errorf("no task with id") + } + spec := "@every " + next.String() + _t := &_task{Task: t, args: args, spec: spec, once: true} + _id, err := c.AddJob(spec, _t) + _t.id = _id + return err +} + +func RunTaskAfter(d time.Duration, id string, args ...interface{}) error { + t := GetTask(id) + if t == nil { + return fmt.Errorf("no task with id") + } + spec := "@every " + d.String() + _t := &_task{Task: t, args: args, spec: spec, once: true} + _id, err := c.AddJob(spec, _t) + _t.id = _id + return err +} + +func Remove(id cron.EntryID) { + c.Remove(id) +} diff --git a/task/task.go b/task/task.go new file mode 100644 index 0000000..86f493b --- /dev/null +++ b/task/task.go @@ -0,0 +1,93 @@ +package task + +import ( + "context" + "fmt" + "time" + + cron "github.com/robfig/cron/v3" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +var l zerolog.Logger +var clog = mylogger{} +var c *cron.Cron + +func init() { + // viper.Set("task.runner", 5) + Reporter = make(chan *Report) +} + +func Init() { + l = log.With().Str("mod", "task").Logger() + c = cron.New( + cron.WithLogger(&clog), + cron.WithChain(func(j cron.Job) cron.Job { + t, ok := j.(*_task) + if ok { + l.Debug().Str("taskname", t.Task.Name).Msg("") + } + return j + }), + cron.WithLocation(time.Local)) +} + +type Task struct { + Name string + ID string + Description string + Group string + + // Func is the stored function that will be called with the args, + // and a interface to carry value will be provided as the first argument + // args will be passed after carry, in order to normalize, + // it is suggested to use strings or other simple types + Func func(c *interface{}, args ...interface{}) error +} + +type _task struct { + Task *Task + id cron.EntryID + args []interface{} + spec string + once bool + // carry value to the next run. Should be modified from the Task.Func + carry *interface{} +} + +func (t *_task) Run() { + if t.carry == nil { + t.carry = new(interface{}) + } + defer func() { + if err := recover(); err != nil { + log.Error().Str("error", fmt.Sprint(err)).Msg("") + } else if t.once { + c.Remove(t.id) + } + }() + if err := t.Task.Func(t.carry, t.args...); err != nil { + log.Error().Err(err).Str("taskname", t.Task.Name).Msg("") + Reporter <- &Report{ + Entry: t.id, + Name: t.Task.Name, + ID: t.Task.ID, + Description: t.Task.Description, + Group: t.Task.Group, + Spec: t.spec, + Once: t.once, + Args: t.args, + Carry: t.carry, + Error: err, + } + } +} + +func Start() { + c.Start() +} + +func Stop() context.Context { + return c.Stop() +} diff --git a/task/task_test.go b/task/task_test.go new file mode 100644 index 0000000..2ff4fd2 --- /dev/null +++ b/task/task_test.go @@ -0,0 +1,124 @@ +package task + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +func TestOverall(t *testing.T) { + zerolog.SetGlobalLevel(zerolog.DebugLevel) + log.Logger = log.Output(zerolog.ConsoleWriter{ + Out: os.Stdout, + TimeFormat: "2006/01/02 15:04:05", + }) + Init() + lg := l.With().Str("test", "TestOverall").Logger() + RunFunc("@every 1s", func() { + p := GetProfile() + d, _ := json.Marshal(p) + lg.Debug().Msg(string(d)) + }) + + RunFunc("@every 2s", func() { l.Debug().Msg("@every 2s") }) + + test1 := &Task{ + Name: "test1", + Description: "test1", + ID: "id1", + Group: "grp1", + Func: func(c *interface{}, args ...interface{}) error { + lg.Debug().Msgf("carry: %v, echo %v", c, args) + return nil + + }, + } + AddTask(test1) + + test2 := &Task{ + Name: "test2", + Description: "test2", + ID: "id2", + Func: func(c *interface{}, args ...interface{}) error { + carry, ok := (*c).(int) + if !ok { + carry = 1 + } + lg.Debug().Msgf("carry: %d, echo %v", carry, args) + carry++ + *c = carry + return nil + }, + } + AddAndRunTask("@every 1s", test2, "1", 2, true) + + RunTask("@every 3s", "id1", "arg1", 2) + RunTaskAfter(time.Second*10, "id2", "ONCE!!!") + RunTaskAt(time.Now().Add(time.Second*15), "id2", "ONCE!Again!!!") + + test3 := &Task{ + Name: "test3", + Description: "test3", + ID: "id3", + Func: func(c *interface{}, args ...interface{}) error { + carry, ok := (*c).(string) + if !ok { + carry = "" + } + lg.Debug().Msgf("carry: %s, echo %v", carry, args) + carry += "c" + *c = carry + return errors.New("test error") + }, + } + AddAndRunTask("@every 5s", test3) + + lg.Info().Msgf("%v", GetTasks()) + Start() + <-time.After(time.Second * 20) + <-Stop().Done() + +} + +func TestReporter(t *testing.T) { + zerolog.SetGlobalLevel(zerolog.DebugLevel) + log.Logger = log.Output(zerolog.ConsoleWriter{ + Out: os.Stdout, + TimeFormat: "2006/01/02 15:04:05", + }) + Init() + AddAndRunTask("@every 2s", &Task{ + Name: "test", + Func: func(c *interface{}, args ...interface{}) error { + carry, ok := (*c).(int) + if !ok { + carry = 0 + } + carry++ + *c = carry + if carry%2 == 1 { + return fmt.Errorf("error with: %d", carry) + } + return nil + }, + }) + + lg := l.With().Str("test", "TestReporter").Logger() + Start() + + go func() { + for { + p := <-Reporter + lg.Debug().Msgf("%+v", p) + } + }() + + <-time.After(time.Second * 20) + <-Stop().Done() +} diff --git a/util/logging.go b/util/logging.go new file mode 100644 index 0000000..80e92ea --- /dev/null +++ b/util/logging.go @@ -0,0 +1,93 @@ +package util + +import ( + "bytes" + "fmt" + "io/ioutil" + "runtime" +) + +func CallerFull(depth int) string { + _, file, line, _ := runtime.Caller(depth) + return fmt.Sprintf("%s:%d", file, line) +} + +func Caller(depth int) string { + _, file, line, _ := runtime.Caller(depth) + short := file + for i := len(file) - 1; i > 0; i-- { + if file[i] == '/' { + short = file[i+1:] + break + } + } + return fmt.Sprintf("%s:%d", short, line) +} + +var ( + dunno = []byte("???") + centerDot = []byte("·") + dot = []byte(".") + slash = []byte("/") +) + +// Stack returns a nicely formatted stack frame, skipping skip frames. +func StackSkip(skip int) string { + buf := new(bytes.Buffer) // the returned data + // As we loop, we open files and read them. These variables record the currently + // loaded file. + var lines [][]byte + var lastFile string + for i := skip; ; i++ { // Skip the expected number of frames + pc, file, line, ok := runtime.Caller(i) + if !ok { + break + } + // Print this much at least. If we can't find the source, it won't show. + fmt.Fprintf(buf, "%s:%d (0x%x)\n", file, line, pc) + if file != lastFile { + data, err := ioutil.ReadFile(file) + if err != nil { + continue + } + lines = bytes.Split(data, []byte{'\n'}) + lastFile = file + } + fmt.Fprintf(buf, "\t%s: %s\n", function(pc), source(lines, line)) + } + return buf.String() +} + +// function returns, if possible, the name of the function containing the PC. +func function(pc uintptr) []byte { + fn := runtime.FuncForPC(pc) + if fn == nil { + return dunno + } + name := []byte(fn.Name()) + // The name includes the path name to the package, which is unnecessary + // since the file name is already included. Plus, it has center dots. + // That is, we see + // runtime/debug.*T·ptrmethod + // and want + // *T.ptrmethod + // Also the package path might contains dot (e.g. code.google.com/...), + // so first eliminate the path prefix + if lastSlash := bytes.LastIndex(name, slash); lastSlash >= 0 { + name = name[lastSlash+1:] + } + if period := bytes.Index(name, dot); period >= 0 { + name = name[period+1:] + } + name = bytes.Replace(name, centerDot, dot, -1) + return name +} + +// source returns a space-trimmed slice of the n'th line. +func source(lines [][]byte, n int) []byte { + n-- // in stack trace, lines are 1-indexed but our array is 0-indexed + if n < 0 || n >= len(lines) { + return dunno + } + return bytes.TrimSpace(lines[n]) +} diff --git a/util/util.go b/util/util.go new file mode 100644 index 0000000..6dc992f --- /dev/null +++ b/util/util.go @@ -0,0 +1,21 @@ +package util + +import ( + "os" + "path/filepath" +) + +func Mkdir(args ...interface{}) error { + var path string + var mode os.FileMode + mode = 0755 + for _, arg := range args { + switch arg := arg.(type) { + case string: + path = filepath.Join(path, arg) + case os.FileMode: + mode = arg + } + } + return os.MkdirAll(path, mode) +}