CL-Softwares
/
modelRT
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

optimize variable naming and optimize real time data computing api

This commit is contained in:
douxu 2025-11-12 17:34:18 +08:00
parent b43adf9b67
commit 041d7e5788
22 changed files with 334 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
constants/buffer.go Normal file
View File

@ -0,0 +1,11 @@
// Package constants define constant variable
package constants
import "time"
const (
// SendMaxBatchSize define maximum buffer capacity
SendMaxBatchSize = 100
// SendMaxBatchInterval define maximum aggregate latency
SendMaxBatchInterval = 200 * time.Millisecond
)

2
handler/alert_event_query.go
View File

@ -35,7 +35,7 @@ func QueryAlertEventHandler(c *gin.Context) {
resp := network.SuccessResponse{
Code: 0,
Msg: "success",
PayLoad: map[string]interface{}{
Payload: map[string]any{
"events": events,
},
}

2
handler/anchor_point_replace.go
View File

@ -68,7 +68,7 @@ func ComponentAnchorReplaceHandler(c *gin.Context) {
resp := network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"uuid": request.UUID,
},
}

4
handler/attr_delete.go
View File

@ -41,7 +41,7 @@ func AttrDeleteHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{"attr_token": request.AttrToken},
Payload: map[string]interface{}{"attr_token": request.AttrToken},
})
return
}
@ -49,7 +49,7 @@ func AttrDeleteHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"attr_token": request.AttrToken,
},
})

4
handler/attr_load.go
View File

@ -46,7 +46,7 @@ func AttrGetHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{"attr_token": request.AttrToken},
Payload: map[string]interface{}{"attr_token": request.AttrToken},
})
return
}
@ -59,7 +59,7 @@ func AttrGetHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"attr_token": request.AttrToken,
"attr_value": attrValue,
},

4
handler/attr_update.go
View File

@ -43,7 +43,7 @@ func AttrSetHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{"attr_token": request.AttrToken},
Payload: map[string]interface{}{"attr_token": request.AttrToken},
})
return
}
@ -51,7 +51,7 @@ func AttrSetHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"attr_token": request.AttrToken,
},
})

12
handler/circuit_diagram_create.go
View File

@ -37,7 +37,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"page_id": request.PageID,
},
}
@ -65,7 +65,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"topologic_info": topologicLink,
},
}
@ -89,7 +89,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"topologic_infos": topologicCreateInfos,
},
}
@ -111,7 +111,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"component_infos": request.ComponentInfos,
},
}
@ -130,7 +130,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"uuid": info.UUID,
"component_params": info.Params,
},
@ -152,7 +152,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) {
resp := network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"page_id": request.PageID,
},
}

16
handler/circuit_diagram_delete.go
View File

@ -42,7 +42,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"page_id": request.PageID,
},
}
@ -70,7 +70,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"topologic_info": topologicLink,
},
}
@ -95,7 +95,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"topologic_info": topologicDelInfo,
},
}
@ -112,7 +112,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"topologic_info": topologicDelInfo,
},
}
@ -138,7 +138,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"uuid": componentInfo.UUID,
},
}
@ -162,7 +162,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"uuid": componentInfo.UUID,
},
}
@ -184,7 +184,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"uuid": componentInfo.UUID,
},
}
@ -205,7 +205,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) {
resp := network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"page_id": request.PageID,
},
}

22
handler/circuit_diagram_load.go
View File

@ -33,7 +33,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"page_id": pageID,
},
}
@ -48,16 +48,16 @@ func CircuitDiagramLoadHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"page_id": pageID,
},
}
c.JSON(http.StatusOK, resp)
return
}
payLoad := make(map[string]interface{})
payLoad["root_vertex"] = topologicInfo.RootVertex
payLoad["topologic"] = topologicInfo.VerticeLinks
payload := make(map[string]interface{})
payload["root_vertex"] = topologicInfo.RootVertex
payload["topologic"] = topologicInfo.VerticeLinks
componentParamMap := make(map[string]any)
for _, VerticeLink := range topologicInfo.VerticeLinks {
@ -69,7 +69,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"uuid": componentUUID,
},
}
@ -84,7 +84,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"uuid": componentUUID,
},
}
@ -103,7 +103,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"uuid": topologicInfo.RootVertex,
},
}
@ -118,7 +118,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"uuid": rootVertexUUID,
},
}
@ -127,12 +127,12 @@ func CircuitDiagramLoadHandler(c *gin.Context) {
}
componentParamMap[rootVertexUUID] = rootComponentParam
payLoad["component_params"] = componentParamMap
payload["component_params"] = componentParamMap
resp := network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: payLoad,
Payload: payload,
}
c.JSON(http.StatusOK, resp)
}

14
handler/circuit_diagram_update.go
View File

@ -35,7 +35,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"page_id": request.PageID,
},
}
@ -52,7 +52,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"topologic_info": topologicLink,
},
}
@ -75,7 +75,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"topologic_info": topologicChangeInfo,
},
}
@ -92,7 +92,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"topologic_info": topologicChangeInfo,
},
}
@ -109,7 +109,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"page_id": request.PageID,
"component_info": request.ComponentInfos,
},
@ -129,7 +129,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) {
resp := network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"uuid": info.UUID,
"component_params": info.Params,
},
@ -152,7 +152,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) {
resp := network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"page_id": request.PageID,
},
}

2
handler/history_data_query.go
View File

@ -50,7 +50,7 @@ func QueryHistoryDataHandler(c *gin.Context) {
resp := network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"events": events,
},
}

6
handler/measurement_load.go
View File

@ -45,7 +45,7 @@ func MeasurementGetHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.FailureResponse{
Code: http.StatusInternalServerError,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"measurement_id": request.MeasurementID,
"measurement_token": request.MeasurementToken,
},
@ -60,7 +60,7 @@ func MeasurementGetHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"measurement_id": request.MeasurementID,
"measurement_token": request.MeasurementToken,
"measurement_value": points,
@ -72,7 +72,7 @@ func MeasurementGetHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: map[string]interface{}{
Payload: map[string]interface{}{
"measurement_id": request.MeasurementID,
"measurement_token": request.MeasurementToken,
"measurement_info": measurementInfo,

10
handler/measurement_recommend.go
View File

@ -17,7 +17,7 @@ import (
// @Tags Measurement Recommend
// @Accept json
// @Produce json
// @Param request body network.MeasurementRecommendRequest true "查询输入参数，例如 'trans' 或 'transformfeeder1_220.'"
// @Param。 input query string true "推荐关键词,例如 'trans' 或 'transformfeeder1_220.'" Example("trans")
// @Success 200 {object} network.SuccessResponse{payload=network.MeasurementRecommendPayload} "返回推荐列表成功"
//
// @Example 200 {
@ -45,8 +45,8 @@ import (
func MeasurementRecommendHandler(c *gin.Context) {
var request network.MeasurementRecommendRequest
if err := c.ShouldBindJSON(&request); err != nil {
logger.Error(c, "failed to unmarshal measurement recommend request", "error", err)
if err := c.ShouldBindQuery(&request); err != nil {
logger.Error(c, "failed to bind measurement recommend request", "error", err)
c.JSON(http.StatusOK, network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
@ -60,7 +60,7 @@ func MeasurementRecommendHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.FailureResponse{
Code: http.StatusInternalServerError,
Msg: err.Error(),
PayLoad: map[string]any{
Payload: map[string]any{
"input": request.Input,
},
})
@ -104,7 +104,7 @@ func MeasurementRecommendHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: &network.MeasurementRecommendPayload{
Payload: &network.MeasurementRecommendPayload{
Input: request.Input,
Offset: finalOffset,
RecommendedList: resultRecommends,

56
handler/real_time_data_pull.go
View File

@ -60,24 +60,51 @@ func PullRealTimeDataHandler(c *gin.Context) {
// TODO[BACKPRESSURE-ISSUE] 先期使用固定大容量对扇入模型进行定义 #1
fanInChan := make(chan network.RealTimePullTarget, 10000)
go processTargetPolling(ctx, globalMonitorState, clientID, fanInChan)
go readClientMessages(ctx, conn, clientID, cancel)
bufferMaxSize := constants.SendMaxBatchSize
sendMaxInterval := constants.SendMaxBatchInterval
buffer := make([]network.RealTimePullTarget, 0, bufferMaxSize)
ticker := time.NewTicker(sendMaxInterval)
defer ticker.Stop()
for {
select {
case targetData, ok := <-fanInChan:
if !ok {
logger.Error(ctx, "fanInChan closed unexpectedly", "clientID", clientID)
logger.Error(ctx, "fanInChan closed unexpectedly", "client_id", clientID)
return
}
// TODO 考虑后续将多个 targets 聚合发送
err := sendRealTimeDataStream(conn, targetData)
if err != nil {
logger.Error(nil, "clientID: %s 写入数据失败: %v", clientID, err)
buffer = append(buffer, targetData)
if len(buffer) >= bufferMaxSize {
// buffer is full, send immediately
if err := sendAggregateRealTimeDataStream(conn, buffer); err != nil {
logger.Error(nil, "when buffer is full, send the real time aggregate data failed", "client_id", clientID, "buffer", buffer, "error", err)
return
}
// reset buffer
buffer = make([]network.RealTimePullTarget, 0, bufferMaxSize)
// reset the ticker to prevent it from triggering immediately after the ticker is sent
ticker.Reset(sendMaxInterval)
}
default:
// TODO[BACKPRESSURE-ISSUE] 考虑在此使用双重背压方式解决阻塞问题 #1
logger.Warn(ctx, "Write channel full, dropping data for slow client.")
case <-ticker.C:
if len(buffer) > 0 {
// when the ticker is triggered, all data in the send buffer is sent
if err := sendAggregateRealTimeDataStream(conn, buffer); err != nil {
logger.Error(nil, "when the ticker is triggered, send the real time aggregate data failed", "client_id", clientID, "buffer", buffer, "error", err)
return
}
// reset buffer
buffer = make([]network.RealTimePullTarget, 0, bufferMaxSize)
}
case <-ctx.Done():
// send the last remaining data
if err := sendAggregateRealTimeDataStream(conn, buffer); err != nil {
logger.Error(nil, "send the last remaining data failed", "client_id", clientID, "buffer", buffer, "error", err)
}
logger.Info(ctx, "PullRealTimeDataHandler exiting as context is done.", "client_id", clientID)
return
}
}
}
@ -107,13 +134,16 @@ func readClientMessages(ctx context.Context, conn *websocket.Conn, clientID stri
}
}
// sendRealTimeDataStream define func to responsible for continuously pushing real-time data to the client
func sendRealTimeDataStream(conn *websocket.Conn, targetData network.RealTimePullTarget) error {
// sendAggregateRealTimeDataStream define func to responsible for continuously pushing aggregate real-time data to the client
func sendAggregateRealTimeDataStream(conn *websocket.Conn, targetsData []network.RealTimePullTarget) error {
if len(targetsData) == 0 {
return nil
}
response := network.SuccessResponse{
Code: 200,
Msg: "success",
PayLoad: network.RealTimePullPayload{
Targets: []network.RealTimePullTarget{targetData},
Payload: network.RealTimePullPayload{
Targets: targetsData,
},
}
return conn.WriteJSON(response)

14
handler/real_time_data_subscription.go
View File

@ -122,7 +122,7 @@ func RealTimeSubHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: network.RealTimeSubPayload{
Payload: network.RealTimeSubPayload{
ClientID: clientID,
TargetResults: results,
},
@ -133,7 +133,7 @@ func RealTimeSubHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: network.RealTimeSubPayload{
Payload: network.RealTimeSubPayload{
ClientID: clientID,
TargetResults: results,
},
@ -146,7 +146,7 @@ func RealTimeSubHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: network.RealTimeSubPayload{
Payload: network.RealTimeSubPayload{
ClientID: clientID,
TargetResults: results,
},
@ -157,7 +157,7 @@ func RealTimeSubHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: network.RealTimeSubPayload{
Payload: network.RealTimeSubPayload{
ClientID: clientID,
TargetResults: results,
},
@ -170,7 +170,7 @@ func RealTimeSubHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: network.RealTimeSubPayload{
Payload: network.RealTimeSubPayload{
ClientID: clientID,
TargetResults: results,
},
@ -181,7 +181,7 @@ func RealTimeSubHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: network.RealTimeSubPayload{
Payload: network.RealTimeSubPayload{
ClientID: clientID,
TargetResults: results,
},
@ -195,7 +195,7 @@ func RealTimeSubHandler(c *gin.Context) {
c.JSON(http.StatusOK, network.FailureResponse{
Code: http.StatusBadRequest,
Msg: err.Error(),
PayLoad: network.RealTimeSubPayload{
Payload: network.RealTimeSubPayload{
ClientID: clientID,
TargetResults: results,
},

19
main.go
View File

@ -22,11 +22,11 @@ import (
"modelRT/middleware"
"modelRT/model"
"modelRT/pool"
realtimedata "modelRT/real-time-data"
"modelRT/router"
"modelRT/util"
realtimedata "modelRT/real-time-data"
"github.com/confluentinc/confluent-kafka-go/kafka"
"github.com/gin-gonic/gin"
"github.com/panjf2000/ants/v2"
swaggerFiles "github.com/swaggo/files"
@ -146,11 +146,18 @@ func main() {
}
defer anchorRealTimePool.Release()
// TODO 配置文件中增加 kafka 配置
// init cancel context
cancelCtx, cancel := context.WithCancel(ctx)
defer cancel()
// init real time data receive channel
go realtimedata.ReceiveChan(cancelCtx)
customerConf := &kafka.ConfigMap{
"bootstrap.servers": modelRTConfig.KafkaConfig.Servers,
"group.id": modelRTConfig.KafkaConfig.GroupID,
"auto.offset.reset": modelRTConfig.KafkaConfig.AutoOffsetReset,
"enable.auto.commit": modelRTConfig.KafkaConfig.EnableAutoCommit,
}
go realtimedata.ReceiveChan(cancelCtx, customerConf, []string{modelRTConfig.KafkaConfig.Topic}, modelRTConfig.KafkaConfig.ReadMessageTimeDuration)
postgresDBClient.Transaction(func(tx *gorm.DB) error {
// load circuit diagram from postgres
@ -170,10 +177,6 @@ func main() {
return nil
})
// TODO 完成订阅数据分析
// TODO 暂时屏蔽完成 swagger 启动测试
// go realtimedata.RealTimeDataComputer(ctx, nil, []string{}, "")
// use release mode in productio
// gin.SetMode(gin.ReleaseMode)
engine := gin.New()

2
network/measurement_request.go
View File

@ -9,5 +9,5 @@ type MeasurementGetRequest struct {
// MeasurementRecommendRequest defines the request payload for an measurement recommend
type MeasurementRecommendRequest struct {
Input string `json:"input" example:"trans"`
Input string `form:"input,omitempty" example:"trans"`
}

4
network/response.go
View File

@ -5,14 +5,14 @@ package network
type FailureResponse struct {
Code int `json:"code" example:"500"`
Msg string `json:"msg" example:"failed to get recommend data from redis"`
PayLoad any `json:"payload" swaggertype:"object"`
Payload any `json:"payload" swaggertype:"object"`
}
// SuccessResponse define struct of standard successful API response format
type SuccessResponse struct {
Code int `json:"code" example:"200"`
Msg string `json:"msg" example:"success"`
PayLoad any `json:"payload" swaggertype:"object"`
Payload any `json:"payload" swaggertype:"object"`
}
// MeasurementRecommendPayload define struct of represents the data payload for the successful recommendation response.

63
real-time-data/kafka.go
View File

@ -1,63 +0,0 @@
// Package realtimedata define real time data operation functions
package realtimedata
import (
"context"
"time"
"modelRT/logger"
"github.com/confluentinc/confluent-kafka-go/kafka"
)
// RealTimeDataComputer continuously processing real-time data from Kafka specified topics
func RealTimeDataComputer(ctx context.Context, consumerConfig kafka.ConfigMap, topics []string, duration string) {
// context for graceful shutdown
ctx, cancel := context.WithCancel(ctx)
defer cancel()
// setup a channel to listen for interrupt signals
// TODO 将中断信号放到入参中
interrupt := make(chan struct{}, 1)
// read message (-1 means wait indefinitely)
timeoutDuration, err := time.ParseDuration(duration)
// create a new consumer
consumer, err := kafka.NewConsumer(&consumerConfig)
if err != nil {
logger.Error(ctx, "init kafka consume by config failed", "config", consumerConfig, "error", err)
}
// subscribe to the topic
err = consumer.SubscribeTopics(topics, nil)
if err != nil {
logger.Error(ctx, "subscribe to the topic failed", "topic", topics, "error", err)
}
// start a goroutine to handle shutdown
go func() {
<-interrupt
cancel()
consumer.Close()
}()
// continuously read messages from Kafka
for {
msg, err := consumer.ReadMessage(timeoutDuration)
if err != nil {
if ctx.Err() == context.Canceled {
logger.Info(ctx, "context canceled, stopping read loop")
break
}
logger.Error(ctx, "consumer read message failed", "error", err)
continue
}
// TODO 使用 ants.pool处理 kafka 的订阅数据
_, err = consumer.CommitMessage(msg)
if err != nil {
logger.Error(ctx, "manual submission information failed", "message", msg, "error", err)
}
}
}

208
real-time-data/real_time_data_computing.go Normal file
View File

@ -0,0 +1,208 @@
// Package realtimedata define real time data operation functions
package realtimedata
import (
"context"
"encoding/json"
"fmt"
"time"
"modelRT/config"
"modelRT/constants"
"modelRT/diagram"
"modelRT/logger"
"modelRT/network"
"modelRT/pool"
"github.com/confluentinc/confluent-kafka-go/kafka"
)
// RealTimeDataChan define channel of real time data receive
var RealTimeDataChan chan network.RealTimeDataReceiveRequest
func init() {
RealTimeDataChan = make(chan network.RealTimeDataReceiveRequest, 100)
}
// ReceiveChan define func of real time data receive and process
func ReceiveChan(ctx context.Context, consumerConfig *kafka.ConfigMap, topics []string, duration string) {
fmt.Println(topics, duration)
consumer, err := kafka.NewConsumer(consumerConfig)
if err != nil {
logger.Error(ctx, "create kafka consumer failed", "error", err)
return
}
defer consumer.Close()
err = consumer.SubscribeTopics(topics, nil)
if err != nil {
logger.Error(ctx, "subscribe kafka topics failed", "topic", topics, "error", err)
return
}
logger.Info(ctx, "start consuming from kafka", "topic", topics)
batchSize := 100
batchTimeout := 2 * time.Second
messages := make([]*kafka.Message, 0, batchSize)
lastCommit := time.Now()
for {
select {
case <-ctx.Done():
return
case realTimeData := <-RealTimeDataChan:
componentUUID := realTimeData.PayLoad.ComponentUUID
component, err := diagram.GetComponentMap(componentUUID)
if err != nil {
logger.Error(ctx, "query component info from diagram map by componet id failed", "component_uuid", componentUUID, "error", err)
continue
}
componentType := component.Type
if componentType != constants.DemoType {
logger.Error(ctx, "can not process real time data of component type not equal DemoType", "component_uuid", componentUUID)
continue
}
var anchorName string
var compareValUpperLimit, compareValLowerLimit float64
var anchorRealTimeData []float64
var calculateFunc func(archorValue float64, args ...float64) float64
// calculateFunc, params := config.SelectAnchorCalculateFuncAndParams(componentType, anchorName, componentData)
for _, param := range realTimeData.PayLoad.Values {
anchorRealTimeData = append(anchorRealTimeData, param.Value)
}
anchorConfig := config.AnchorParamConfig{
AnchorParamBaseConfig: config.AnchorParamBaseConfig{
ComponentUUID: componentUUID,
AnchorName: anchorName,
CompareValUpperLimit: compareValUpperLimit,
CompareValLowerLimit: compareValLowerLimit,
AnchorRealTimeData: anchorRealTimeData,
},
CalculateFunc: calculateFunc,
CalculateParams: []float64{},
}
anchorChan, err := pool.GetAnchorParamChan(ctx, componentUUID)
if err != nil {
logger.Error(ctx, "get anchor param chan failed", "component_uuid", componentUUID, "error", err)
continue
}
anchorChan <- anchorConfig
default:
msg, err := consumer.ReadMessage(batchTimeout)
if err != nil {
if err.(kafka.Error).Code() == kafka.ErrTimedOut {
// 超时时处理累积的消息
if len(messages) > 0 {
processMessageBatch(ctx, messages)
consumer.Commit()
messages = messages[:0]
}
continue
}
logger.Error(ctx, "read message from kafka failed", "error", err)
continue
}
messages = append(messages, msg)
// TODO 达到批处理大小或超时时间时处理消息
if len(messages) >= batchSize || time.Since(lastCommit) >= batchTimeout {
processMessageBatch(ctx, messages)
consumer.Commit()
messages = messages[:0]
lastCommit = time.Now()
}
}
}
}
type RealTimeDataPayload struct {
ComponentUUID string
Values []float64
}
type RealTimeData struct {
Payload RealTimeDataPayload
}
func parseKafkaMessage(msgValue []byte) (*RealTimeData, error) {
var realTimeData RealTimeData
err := json.Unmarshal(msgValue, &realTimeData)
if err != nil {
return nil, fmt.Errorf("json unmarshal failed: %w", err)
}
return &realTimeData, nil
}
func processRealTimeData(ctx context.Context, realTimeData *RealTimeData) {
componentUUID := realTimeData.Payload.ComponentUUID
component, err := diagram.GetComponentMap(componentUUID)
if err != nil {
logger.Error(ctx, "query component info from diagram map by component id failed",
"component_uuid", componentUUID, "error", err)
return
}
componentType := component.Type
if componentType != constants.DemoType {
logger.Error(ctx, "can not process real time data of component type not equal DemoType",
"component_uuid", componentUUID)
return
}
var anchorName string
var compareValUpperLimit, compareValLowerLimit float64
var anchorRealTimeData []float64
var calculateFunc func(archorValue float64, args ...float64) float64
// 收集实时数据
for _, param := range realTimeData.Payload.Values {
anchorRealTimeData = append(anchorRealTimeData, param)
}
anchorConfig := config.AnchorParamConfig{
AnchorParamBaseConfig: config.AnchorParamBaseConfig{
ComponentUUID: componentUUID,
AnchorName: anchorName,
CompareValUpperLimit: compareValUpperLimit,
CompareValLowerLimit: compareValLowerLimit,
AnchorRealTimeData: anchorRealTimeData,
},
CalculateFunc: calculateFunc,
CalculateParams: []float64{},
}
anchorChan, err := pool.GetAnchorParamChan(ctx, componentUUID)
if err != nil {
logger.Error(ctx, "get anchor param chan failed",
"component_uuid", componentUUID, "error", err)
return
}
// TODO 使用select避免channel阻塞
select {
case anchorChan <- anchorConfig:
// 成功发送
case <-ctx.Done():
logger.Info(ctx, "context done while sending to anchor chan")
case <-time.After(5 * time.Second):
logger.Error(ctx, "timeout sending to anchor chan", "component_uuid", componentUUID)
}
}
func processMessageBatch(ctx context.Context, messages []*kafka.Message) {
for _, msg := range messages {
realTimeData, err := parseKafkaMessage(msg.Value)
if err != nil {
logger.Error(ctx, "parse kafka message failed", "error", err)
continue
}
processRealTimeData(ctx, realTimeData)
}
}

73
real-time-data/real_time_data_receive.go
View File

@ -1,73 +0,0 @@
// Package realtimedata define real time data operation functions
package realtimedata
import (
"context"
"modelRT/config"
"modelRT/constants"
"modelRT/diagram"
"modelRT/logger"
"modelRT/network"
"modelRT/pool"
)
// RealTimeDataChan define channel of real time data receive
var RealTimeDataChan chan network.RealTimeDataReceiveRequest
func init() {
RealTimeDataChan = make(chan network.RealTimeDataReceiveRequest, 100)
}
// ReceiveChan define func of real time data receive and process
func ReceiveChan(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case realTimeData := <-RealTimeDataChan:
componentUUID := realTimeData.PayLoad.ComponentUUID
component, err := diagram.GetComponentMap(componentUUID)
if err != nil {
logger.Error(ctx, "query component info from diagram map by componet id failed", "component_uuid", componentUUID, "error", err)
continue
}
componentType := component.Type
if componentType != constants.DemoType {
logger.Error(ctx, "can not process real time data of component type not equal DemoType", "component_uuid", componentUUID)
continue
}
var anchorName string
var compareValUpperLimit, compareValLowerLimit float64
var anchorRealTimeData []float64
var calculateFunc func(archorValue float64, args ...float64) float64
// calculateFunc, params := config.SelectAnchorCalculateFuncAndParams(componentType, anchorName, componentData)
for _, param := range realTimeData.PayLoad.Values {
anchorRealTimeData = append(anchorRealTimeData, param.Value)
}
anchorConfig := config.AnchorParamConfig{
AnchorParamBaseConfig: config.AnchorParamBaseConfig{
ComponentUUID: componentUUID,
AnchorName: anchorName,
CompareValUpperLimit: compareValUpperLimit,
CompareValLowerLimit: compareValLowerLimit,
AnchorRealTimeData: anchorRealTimeData,
},
CalculateFunc: calculateFunc,
CalculateParams: []float64{},
}
anchorChan, err := pool.GetAnchorParamChan(ctx, componentUUID)
if err != nil {
logger.Error(ctx, "get anchor param chan failed", "component_uuid", componentUUID, "error", err)
continue
}
anchorChan <- anchorConfig
default:
}
}
}

4
router/monitor.go
View File

@ -10,6 +10,6 @@ import (
// registerMonitorRoutes define func of register monitordata routes
func registerMonitorRoutes(rg *gin.RouterGroup) {
g := rg.Group("/monitors/")
g.POST("/data/subscriptions", handler.RealTimeSubHandler)
g.GET("/data/realtime/stream/:clientID", handler.PullRealTimeDataHandler)
g.POST("data/subscriptions", handler.RealTimeSubHandler)
g.GET("data/realtime/stream/:clientID", handler.PullRealTimeDataHandler)
}