From ca3d19a5d7ce6fd88c98e7f92178eb13edc538b1 Mon Sep 17 00:00:00 2001 From: 0xJacky Date: Thu, 18 May 2023 15:08:26 +0800 Subject: [PATCH] enhance: node analytic init --- frontend/.yarnrc.yml | 2 + frontend/package.json | 3 +- frontend/src/version.json | 2 +- frontend/version.json | 2 +- main.go | 3 +- server/api/analytic.go | 4 +- server/api/environment.go | 4 +- server/api/node.go | 30 +-- server/api/template.go | 16 +- server/api/upgrade.go | 14 +- server/internal/analytic/node.go | 194 ++++++------------ server/internal/analytic/node_record.go | 102 +++++++++ server/internal/analytic/node_stat.go | 61 ++++++ server/internal/environment/environment.go | 22 ++ .../template}/template.go | 2 +- .../{service => internal/upgrader}/upgrade.go | 2 +- server/server.go | 5 + server/service/environment.go | 97 --------- 18 files changed, 301 insertions(+), 264 deletions(-) create mode 100644 server/internal/analytic/node_record.go create mode 100644 server/internal/analytic/node_stat.go rename server/{service => internal/template}/template.go (99%) rename server/{service => internal/upgrader}/upgrade.go (99%) delete mode 100644 server/service/environment.go diff --git a/frontend/.yarnrc.yml b/frontend/.yarnrc.yml index 3186f3f0..fdc343f9 100644 --- a/frontend/.yarnrc.yml +++ b/frontend/.yarnrc.yml @@ -1 +1,3 @@ nodeLinker: node-modules + +yarnPath: .yarn/releases/yarn-1.22.19.cjs diff --git a/frontend/package.json b/frontend/package.json index 73795d61..35466b36 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -50,5 +50,6 @@ "vite-plugin-html": "^3.2.0", "vite-svg-loader": "^4.0.0", "vue-tsc": "^1.6.1" - } + }, + "packageManager": "yarn@1.22.19" } diff --git a/frontend/src/version.json b/frontend/src/version.json index 81464217..6bfae0d1 100644 --- a/frontend/src/version.json +++ b/frontend/src/version.json @@ -1 +1 @@ -{"version":"1.9.9","build_id":127,"total_build":197} \ No newline at end of file +{"version":"1.9.9","build_id":129,"total_build":199} \ No newline at end of file diff --git a/frontend/version.json b/frontend/version.json index 81464217..6bfae0d1 100644 --- a/frontend/version.json +++ b/frontend/version.json @@ -1 +1 @@ -{"version":"1.9.9","build_id":127,"total_build":197} \ No newline at end of file +{"version":"1.9.9","build_id":129,"total_build":199} \ No newline at end of file diff --git a/main.go b/main.go index a588d57d..e97ab2db 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ import ( "flag" "fmt" "github.com/0xJacky/Nginx-UI/server" - "github.com/0xJacky/Nginx-UI/server/service" "github.com/0xJacky/Nginx-UI/server/settings" "github.com/gin-gonic/gin" "github.com/jpillora/overseer" @@ -21,7 +20,7 @@ func main() { gin.SetMode(settings.ServerSettings.RunMode) - r, err := service.GetRuntimeInfo() + r, err := server.GetRuntimeInfo() if err != nil { log.Fatalln(err) diff --git a/server/api/analytic.go b/server/api/analytic.go index 542a153d..5753549c 100644 --- a/server/api/analytic.go +++ b/server/api/analytic.go @@ -230,13 +230,13 @@ func GetIntroAnalytic(c *gin.Context) { for { // write - err = ws.WriteJSON(analytic.GetNodeAnalyticIntro()) + err = ws.WriteJSON(analytic.GetNodeStat()) if err != nil { logger.Error(err) break } - time.Sleep(5 * time.Second) + time.Sleep(10 * time.Second) } } diff --git a/server/api/environment.go b/server/api/environment.go index a67bef6c..32538bfb 100644 --- a/server/api/environment.go +++ b/server/api/environment.go @@ -1,9 +1,9 @@ package api import ( + "github.com/0xJacky/Nginx-UI/server/internal/environment" "github.com/0xJacky/Nginx-UI/server/model" "github.com/0xJacky/Nginx-UI/server/query" - "github.com/0xJacky/Nginx-UI/server/service" "github.com/gin-gonic/gin" "github.com/spf13/cast" "net/http" @@ -24,7 +24,7 @@ func GetEnvironment(c *gin.Context) { } func GetEnvironmentList(c *gin.Context) { - data, err := service.RetrieveEnvironmentList() + data, err := environment.RetrieveEnvironmentList() if err != nil { ErrHandler(c, err) return diff --git a/server/api/node.go b/server/api/node.go index 09d1d6e0..ac36b17c 100644 --- a/server/api/node.go +++ b/server/api/node.go @@ -2,7 +2,7 @@ package api import ( "github.com/0xJacky/Nginx-UI/server/internal/analytic" - "github.com/0xJacky/Nginx-UI/server/service" + "github.com/0xJacky/Nginx-UI/server/internal/upgrader" "github.com/dustin/go-humanize" "github.com/gin-gonic/gin" "github.com/shirou/gopsutil/v3/cpu" @@ -18,28 +18,28 @@ func GetCurrentNode(c *gin.Context) { return } - runtimeInfo, err := service.GetRuntimeInfo() + runtimeInfo, err := upgrader.GetRuntimeInfo() if err != nil { ErrHandler(c, err) return } - cpuInfo, _ := cpu.Info() memory, _ := getMemoryStat() - ver, _ := service.GetCurrentVersion() + ver, _ := upgrader.GetCurrentVersion() diskUsage, _ := disk.Usage(".") - intro := analytic.GetNodeAnalyticIntro() - - nodeInfo := service.NodeInfo{ - RequestNodeSecret: c.MustGet("NodeSecret").(string), - NodeRuntimeInfo: runtimeInfo, - CPUNum: len(cpuInfo), - MemoryTotal: memory.Total, - DiskTotal: humanize.Bytes(diskUsage.Total), - Version: ver.Version, - Node: intro, + nodeInfo := analytic.NodeInfo{ + NodeRuntimeInfo: runtimeInfo, + CPUNum: len(cpuInfo), + MemoryTotal: memory.Total, + DiskTotal: humanize.Bytes(diskUsage.Total), + Version: ver.Version, } - c.JSON(http.StatusOK, nodeInfo) + stat := analytic.GetNodeStat() + + c.JSON(http.StatusOK, analytic.Node{ + NodeInfo: nodeInfo, + NodeStat: stat, + }) } diff --git a/server/api/template.go b/server/api/template.go index cab7921e..38c1a5e8 100644 --- a/server/api/template.go +++ b/server/api/template.go @@ -2,7 +2,7 @@ package api import ( "github.com/0xJacky/Nginx-UI/server/internal/nginx" - "github.com/0xJacky/Nginx-UI/server/service" + "github.com/0xJacky/Nginx-UI/server/internal/template" "github.com/gin-gonic/gin" "net/http" ) @@ -45,7 +45,7 @@ func GetTemplate(c *gin.Context) { } func GetTemplateConfList(c *gin.Context) { - configList, err := service.GetTemplateList("conf") + configList, err := template.GetTemplateList("conf") if err != nil { ErrHandler(c, err) @@ -58,7 +58,7 @@ func GetTemplateConfList(c *gin.Context) { } func GetTemplateBlockList(c *gin.Context) { - configList, err := service.GetTemplateList("block") + configList, err := template.GetTemplateList("block") if err != nil { ErrHandler(c, err) @@ -72,18 +72,18 @@ func GetTemplateBlockList(c *gin.Context) { func GetTemplateBlock(c *gin.Context) { type resp struct { - service.ConfigInfoItem - service.ConfigDetail + template.ConfigInfoItem + template.ConfigDetail } - var bindData map[string]service.TVariable + var bindData map[string]template.TVariable _ = c.ShouldBindJSON(&bindData) - info := service.GetTemplateInfo("block", c.Param("name")) + info := template.GetTemplateInfo("block", c.Param("name")) if bindData == nil { bindData = info.Variables } - detail, err := service.ParseTemplate("block", c.Param("name"), bindData) + detail, err := template.ParseTemplate("block", c.Param("name"), bindData) if err != nil { ErrHandler(c, err) return diff --git a/server/api/upgrade.go b/server/api/upgrade.go index d016bd7f..cf78778c 100644 --- a/server/api/upgrade.go +++ b/server/api/upgrade.go @@ -2,7 +2,7 @@ package api import ( "github.com/0xJacky/Nginx-UI/server/internal/logger" - "github.com/0xJacky/Nginx-UI/server/service" + "github.com/0xJacky/Nginx-UI/server/internal/upgrader" "github.com/gin-gonic/gin" "github.com/gorilla/websocket" "net/http" @@ -10,19 +10,19 @@ import ( ) func GetRelease(c *gin.Context) { - data, err := service.GetRelease(c.Query("channel")) + data, err := upgrader.GetRelease(c.Query("channel")) if err != nil { ErrHandler(c, err) return } - runtimeInfo, err := service.GetRuntimeInfo() + runtimeInfo, err := upgrader.GetRuntimeInfo() if err != nil { ErrHandler(c, err) return } type resp struct { - service.TRelease - service.RuntimeInfo + upgrader.TRelease + upgrader.RuntimeInfo } c.JSON(http.StatusOK, resp{ data, runtimeInfo, @@ -30,7 +30,7 @@ func GetRelease(c *gin.Context) { } func GetCurrentVersion(c *gin.Context) { - curVer, err := service.GetCurrentVersion() + curVer, err := upgrader.GetCurrentVersion() if err != nil { ErrHandler(c, err) return @@ -70,7 +70,7 @@ func PerformCoreUpgrade(c *gin.Context) { "message": "Initialing core upgrader", }) - u, err := service.NewUpgrader(control.Channel) + u, err := upgrader.NewUpgrader(control.Channel) if err != nil { _ = ws.WriteJSON(gin.H{ diff --git a/server/internal/analytic/node.go b/server/internal/analytic/node.go index 3ed00ed3..bf8671cf 100644 --- a/server/internal/analytic/node.go +++ b/server/internal/analytic/node.go @@ -1,31 +1,43 @@ package analytic import ( + "crypto/tls" "encoding/json" "github.com/0xJacky/Nginx-UI/server/internal/logger" + "github.com/0xJacky/Nginx-UI/server/internal/upgrader" "github.com/0xJacky/Nginx-UI/server/model" - "github.com/0xJacky/Nginx-UI/server/query" - "github.com/gorilla/websocket" - "github.com/opentracing/opentracing-go/log" - "github.com/shirou/gopsutil/v3/cpu" "github.com/shirou/gopsutil/v3/load" "github.com/shirou/gopsutil/v3/net" - "math" + "io" "net/http" - "runtime" + "net/url" "sync" "time" ) -type Node struct { - EnvironmentID int `json:"environment_id,omitempty"` - Name string `json:"name,omitempty"` +type NodeInfo struct { + NodeRuntimeInfo upgrader.RuntimeInfo `json:"node_runtime_info"` + Version string `json:"version"` + CPUNum int `json:"cpu_num"` + MemoryTotal string `json:"memory_total"` + DiskTotal string `json:"disk_total"` +} + +type NodeStat struct { AvgLoad *load.AvgStat `json:"avg_load"` CPUPercent float64 `json:"cpu_percent"` MemoryPercent float64 `json:"memory_percent"` DiskPercent float64 `json:"disk_percent"` Network net.IOCountersStat `json:"network"` Status bool `json:"status"` + ResponseAt time.Time `json:"response_at"` +} + +type Node struct { + EnvironmentID int `json:"environment_id,omitempty"` + *model.Environment + NodeStat + NodeInfo } var mutex sync.Mutex @@ -38,136 +50,66 @@ func init() { NodeMap = make(TNodeMap) } -func nodeAnalyticLive(env *model.Environment, errChan chan error) { - for { - err := nodeAnalyticRecord(env) - - if err != nil { - // set node offline - if NodeMap[env.ID] != nil { - NodeMap[env.ID].Status = false - } - log.Error(err) - errChan <- err - // wait 5s then reconnect - time.Sleep(5 * time.Second) +func GetNode(env *model.Environment) (n *Node) { + if env == nil { + logger.Error("env is nil") + return + } + n, ok := NodeMap[env.ID] + if !ok { + n = &Node{ + Environment: env, } } + + return n } -func nodeAnalyticRecord(env *model.Environment) (err error) { - url, err := env.GetWebSocketURL("/api/analytic/intro") +func InitNode(env *model.Environment) (n *Node) { + n = &Node{ + Environment: env, + } + + u, err := url.JoinPath(env.URL, "/api/node") if err != nil { + logger.Error(err) return } - header := http.Header{} - - header.Set("X-Node-Secret", env.Token) - - c, _, err := websocket.DefaultDialer.Dial(url, header) if err != nil { + logger.Error(err) + return + } + client := http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + req, err := http.NewRequest("GET", u, nil) + req.Header.Set("X-Node-Secret", env.Token) + + resp, err := client.Do(req) + + if err != nil { + logger.Error(err) return } - defer c.Close() + defer resp.Body.Close() + bytes, _ := io.ReadAll(resp.Body) - for { - _, message, err := c.ReadMessage() - if err != nil { - return err - } - logger.Debugf("recv: %s %s", env.Name, message) - - var nodeAnalytic Node - - err = json.Unmarshal(message, &nodeAnalytic) - - if err != nil { - return err - } - - nodeAnalytic.EnvironmentID = env.ID - nodeAnalytic.Name = env.Name - // set online - nodeAnalytic.Status = true - mutex.Lock() - NodeMap[env.ID] = &nodeAnalytic - mutex.Unlock() - } -} - -func RetrieveNodesStatus() { - NodeMap = make(TNodeMap) - - env := query.Environment - - envs, err := env.Find() - - if err != nil { - logger.Error(err) - return - } - - errChan := make(chan error) - - for _, v := range envs { - go nodeAnalyticLive(v, errChan) - } - - // block at here - for err = range errChan { - log.Error(err) - } -} - -func GetNodeAnalyticIntro() (data Node) { - memory, err := GetMemoryStat() - - if err != nil { - logger.Error(err) - return - } - - cpuTimesBefore, _ := cpu.Times(false) - time.Sleep(1000 * time.Millisecond) - cpuTimesAfter, _ := cpu.Times(false) - threadNum := runtime.GOMAXPROCS(0) - cpuUserUsage := (cpuTimesAfter[0].User - cpuTimesBefore[0].User) / (float64(1000*threadNum) / 1000) - cpuSystemUsage := (cpuTimesAfter[0].System - cpuTimesBefore[0].System) / (float64(1000*threadNum) / 1000) - - loadAvg, err := load.Avg() - - if err != nil { - logger.Error(err) - return - } - - diskStat, err := GetDiskStat() - - if err != nil { - logger.Error(err) - return - } - - netIO, err := net.IOCounters(false) - - if err != nil { - logger.Error(err) - return - } - - var network net.IOCountersStat - if len(netIO) > 0 { - network = netIO[0] - } - - return Node{ - AvgLoad: loadAvg, - CPUPercent: math.Min((cpuUserUsage+cpuSystemUsage)*100, 100), - MemoryPercent: memory.Pressure, - DiskPercent: diskStat.Percentage, - Network: network, + if resp.StatusCode != 200 { + logger.Error(string(bytes)) + return } + + logger.Debug(string(bytes)) + err = json.Unmarshal(bytes, &n.NodeInfo) + if err != nil { + logger.Error(err) + return + } + + return } diff --git a/server/internal/analytic/node_record.go b/server/internal/analytic/node_record.go new file mode 100644 index 00000000..d1db6fc0 --- /dev/null +++ b/server/internal/analytic/node_record.go @@ -0,0 +1,102 @@ +package analytic + +import ( + "encoding/json" + "github.com/0xJacky/Nginx-UI/server/internal/logger" + "github.com/0xJacky/Nginx-UI/server/model" + "github.com/0xJacky/Nginx-UI/server/query" + "github.com/gorilla/websocket" + "github.com/opentracing/opentracing-go/log" + "net/http" + "time" +) + +func RetrieveNodesStatus() { + NodeMap = make(TNodeMap) + + env := query.Environment + + envs, err := env.Find() + + if err != nil { + logger.Error(err) + return + } + + errChan := make(chan error) + + for _, v := range envs { + go nodeAnalyticLive(v, errChan) + } + + // block at here + for err = range errChan { + log.Error(err) + } +} + +func nodeAnalyticLive(env *model.Environment, errChan chan error) { + for { + err := nodeAnalyticRecord(env) + + if err != nil { + // set node offline + if NodeMap[env.ID] != nil { + mutex.Lock() + NodeMap[env.ID].Status = false + mutex.Unlock() + } + logger.Error(err) + errChan <- err + // wait 5s then reconnect + time.Sleep(5 * time.Second) + } + } +} + +func nodeAnalyticRecord(env *model.Environment) (err error) { + mutex.Lock() + NodeMap[env.ID] = InitNode(env) + mutex.Unlock() + + u, err := env.GetWebSocketURL("/api/analytic/intro") + + if err != nil { + return + } + + header := http.Header{} + + header.Set("X-Node-Secret", env.Token) + + c, _, err := websocket.DefaultDialer.Dial(u, header) + if err != nil { + return + } + + defer c.Close() + + var nodeStat NodeStat + + for { + _, message, err := c.ReadMessage() + if err != nil { + return err + } + logger.Debugf("recv: %s %s", env.Name, message) + + err = json.Unmarshal(message, &nodeStat) + + if err != nil { + return err + } + + // set online + nodeStat.Status = true + nodeStat.ResponseAt = time.Now() + + mutex.Lock() + NodeMap[env.ID].NodeStat = nodeStat + mutex.Unlock() + } +} diff --git a/server/internal/analytic/node_stat.go b/server/internal/analytic/node_stat.go new file mode 100644 index 00000000..4f5aba67 --- /dev/null +++ b/server/internal/analytic/node_stat.go @@ -0,0 +1,61 @@ +package analytic + +import ( + "github.com/0xJacky/Nginx-UI/server/internal/logger" + "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v3/net" + "math" + "runtime" + "time" +) + +func GetNodeStat() (data NodeStat) { + memory, err := GetMemoryStat() + + if err != nil { + logger.Error(err) + return + } + + cpuTimesBefore, _ := cpu.Times(false) + time.Sleep(1000 * time.Millisecond) + cpuTimesAfter, _ := cpu.Times(false) + threadNum := runtime.GOMAXPROCS(0) + cpuUserUsage := (cpuTimesAfter[0].User - cpuTimesBefore[0].User) / (float64(1000*threadNum) / 1000) + cpuSystemUsage := (cpuTimesAfter[0].System - cpuTimesBefore[0].System) / (float64(1000*threadNum) / 1000) + + loadAvg, err := load.Avg() + + if err != nil { + logger.Error(err) + return + } + + diskStat, err := GetDiskStat() + + if err != nil { + logger.Error(err) + return + } + + netIO, err := net.IOCounters(false) + + if err != nil { + logger.Error(err) + return + } + + var network net.IOCountersStat + if len(netIO) > 0 { + network = netIO[0] + } + + return NodeStat{ + AvgLoad: loadAvg, + CPUPercent: math.Min((cpuUserUsage+cpuSystemUsage)*100, 100), + MemoryPercent: memory.Pressure, + DiskPercent: diskStat.Percentage, + Network: network, + } +} diff --git a/server/internal/environment/environment.go b/server/internal/environment/environment.go index f2d9ed19..c7cf6ea2 100644 --- a/server/internal/environment/environment.go +++ b/server/internal/environment/environment.go @@ -1 +1,23 @@ package environment + +import ( + "github.com/0xJacky/Nginx-UI/server/internal/analytic" + "github.com/0xJacky/Nginx-UI/server/query" +) + +func RetrieveEnvironmentList() (envs []*analytic.Node, err error) { + envQuery := query.Environment + + data, err := envQuery.Find() + if err != nil { + return + } + + for _, v := range data { + t := analytic.GetNode(v) + + envs = append(envs, t) + } + + return +} diff --git a/server/service/template.go b/server/internal/template/template.go similarity index 99% rename from server/service/template.go rename to server/internal/template/template.go index 6f567ad5..b12d61df 100644 --- a/server/service/template.go +++ b/server/internal/template/template.go @@ -1,4 +1,4 @@ -package service +package template import ( "bufio" diff --git a/server/service/upgrade.go b/server/internal/upgrader/upgrade.go similarity index 99% rename from server/service/upgrade.go rename to server/internal/upgrader/upgrade.go index 1e3f115c..a461243b 100644 --- a/server/service/upgrade.go +++ b/server/internal/upgrader/upgrade.go @@ -1,4 +1,4 @@ -package service +package upgrader import ( "encoding/json" diff --git a/server/server.go b/server/server.go index 166a6775..6ee5cd8c 100644 --- a/server/server.go +++ b/server/server.go @@ -5,6 +5,7 @@ import ( "github.com/0xJacky/Nginx-UI/server/internal/cert" "github.com/0xJacky/Nginx-UI/server/internal/logger" "github.com/0xJacky/Nginx-UI/server/internal/nginx" + "github.com/0xJacky/Nginx-UI/server/internal/upgrader" "github.com/0xJacky/Nginx-UI/server/model" "github.com/0xJacky/Nginx-UI/server/query" "github.com/0xJacky/Nginx-UI/server/router" @@ -18,6 +19,10 @@ import ( "time" ) +func GetRuntimeInfo() (r upgrader.RuntimeInfo, err error) { + return upgrader.GetRuntimeInfo() +} + func Program(state overseer.State) { defer logger.Sync() // Hack: fix wrong Content Type of .js file on some OS platforms diff --git a/server/service/environment.go b/server/service/environment.go deleted file mode 100644 index e109f352..00000000 --- a/server/service/environment.go +++ /dev/null @@ -1,97 +0,0 @@ -package service - -import ( - "crypto/tls" - "encoding/json" - "github.com/0xJacky/Nginx-UI/server/internal/analytic" - "github.com/0xJacky/Nginx-UI/server/internal/logger" - "github.com/0xJacky/Nginx-UI/server/model" - "github.com/0xJacky/Nginx-UI/server/query" - "io" - "net/http" - "net/url" - "time" -) - -type Environment struct { - *model.Environment - Status bool `json:"status"` - NodeInfo -} - -func RetrieveEnvironmentList() (envs []*Environment, err error) { - envQuery := query.Environment - - data, err := envQuery.Find() - if err != nil { - return - } - - for _, v := range data { - t := &Environment{ - Environment: v, - } - - node, status := t.GetNode() - t.Status = status - t.NodeInfo = node - - envs = append(envs, t) - } - - return -} - -type NodeInfo struct { - RequestNodeSecret string `json:"request_node_secret"` - NodeRuntimeInfo RuntimeInfo `json:"node_runtime_info"` - Version string `json:"version"` - CPUNum int `json:"cpu_num"` - MemoryTotal string `json:"memory_total"` - DiskTotal string `json:"disk_total"` - ResponseAt time.Time `json:"response_at"` - analytic.Node -} - -func (env *Environment) GetNode() (node NodeInfo, status bool) { - u, err := url.JoinPath(env.URL, "/api/node") - - if err != nil { - logger.Error(err) - return - } - client := http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - } - req, err := http.NewRequest("GET", u, nil) - req.Header.Set("X-Node-Secret", env.Token) - - resp, err := client.Do(req) - - if err != nil { - logger.Error(err) - return - } - - defer resp.Body.Close() - bytes, _ := io.ReadAll(resp.Body) - - if resp.StatusCode != 200 { - logger.Error(string(bytes)) - return - } - - logger.Debug(string(bytes)) - err = json.Unmarshal(bytes, &node) - if err != nil { - logger.Error(err) - return - } - - node.ResponseAt = time.Now() - status = true - - return -}