refactor: nodes analytics (#847)

* refactor: nodes analytics

* feat(debug): add pprof in debug mode

* refactor: websocket error handler
This commit is contained in:
Jacky 2025-02-05 17:25:29 +08:00
parent b1ba719cb1
commit cb4977e5ab
No known key found for this signature in database
GPG key ID: 215C21B10DF38B4D
17 changed files with 276 additions and 155 deletions

View file

@ -2,81 +2,112 @@ package analytic
import (
"context"
"encoding/json"
"net/http"
"sync"
"time"
"github.com/0xJacky/Nginx-UI/model"
"github.com/0xJacky/Nginx-UI/query"
"github.com/gorilla/websocket"
"github.com/uozi-tech/cosy/logger"
"net/http"
"time"
)
var stopNodeRecordChan = make(chan struct{})
var (
ctx, cancel = context.WithCancel(context.Background())
wg sync.WaitGroup
restartMu sync.Mutex // Add mutex to prevent concurrent restarts
)
func RestartRetrieveNodesStatus() {
stopNodeRecordChan <- struct{}{}
time.Sleep(10 * time.Second)
go RetrieveNodesStatus()
restartMu.Lock() // Acquire lock before modifying shared resources
defer restartMu.Unlock()
// Cancel previous context to stop all operations
cancel()
// Wait for previous goroutines to finish
wg.Wait()
// Create new context for this run
ctx, cancel = context.WithCancel(context.Background())
wg.Add(1)
go func() {
defer wg.Done()
RetrieveNodesStatus()
}()
}
func RetrieveNodesStatus() {
NodeMap = make(TNodeMap)
errChan := make(chan error)
logger.Info("RetrieveNodesStatus start")
defer logger.Info("RetrieveNodesStatus exited")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
mutex.Lock()
if NodeMap == nil {
NodeMap = make(TNodeMap)
}
mutex.Unlock()
env := query.Environment
envs, err := env.Where(env.Enabled.Is(true)).Find()
if err != nil {
logger.Error(err)
return
}
for _, v := range envs {
go nodeAnalyticLive(v, errChan, ctx)
}
var wg sync.WaitGroup
defer wg.Wait()
for {
select {
case err = <-errChan:
logger.Error(err)
case <-stopNodeRecordChan:
logger.Info("RetrieveNodesStatus exited normally")
return // will execute defer cancel()
}
}
}
for _, env := range envs {
wg.Add(1)
go func(e *model.Environment) {
defer wg.Done()
retryTicker := time.NewTicker(5 * time.Second)
defer retryTicker.Stop()
func nodeAnalyticLive(env *model.Environment, errChan chan error, ctx context.Context) {
for {
err := nodeAnalyticRecord(env, ctx)
if err != nil {
// set node offline
if NodeMap[env.ID] != nil {
mutex.Lock()
NodeMap[env.ID].Status = false
mutex.Unlock()
for {
select {
case <-ctx.Done():
return
default:
if err := nodeAnalyticRecord(e, ctx); err != nil {
logger.Error(err)
if NodeMap[env.ID] != nil {
mutex.Lock()
NodeMap[env.ID].Status = false
mutex.Unlock()
}
select {
case <-retryTicker.C:
case <-ctx.Done():
return
}
}
}
}
logger.Error(err)
errChan <- err
// wait 5s then reconnect
time.Sleep(5 * time.Second)
}
}(env)
}
<-ctx.Done()
}
func nodeAnalyticRecord(env *model.Environment, ctx context.Context) (err error) {
func nodeAnalyticRecord(env *model.Environment, ctx context.Context) error {
scopeCtx, cancel := context.WithCancel(ctx)
defer cancel()
node, err := InitNode(env)
mutex.Lock()
NodeMap[env.ID] = InitNode(env)
NodeMap[env.ID] = node
mutex.Unlock()
if err != nil {
return err
}
u, err := env.GetWebSocketURL("/api/analytic/intro")
if err != nil {
return
return err
}
header := http.Header{}
@ -90,28 +121,20 @@ func nodeAnalyticRecord(env *model.Environment, ctx context.Context) (err error)
c, _, err := dial.Dial(u, header)
if err != nil {
return
return err
}
defer c.Close()
var nodeStat NodeStat
go func() {
// shutdown
<-ctx.Done()
<-scopeCtx.Done()
_ = c.Close()
}()
var nodeStat NodeStat
for {
_, message, err := c.ReadMessage()
if err != nil || websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNoStatusReceived,
websocket.CloseNormalClosure) {
return err
}
err = json.Unmarshal(message, &nodeStat)
err = c.ReadJSON(&nodeStat)
if err != nil {
return err
}