From fd2b202037445be8addf48afd96f15f27a943de9 Mon Sep 17 00:00:00 2001 From: douxu Date: Thu, 22 Jan 2026 16:19:00 +0800 Subject: [PATCH 01/43] optimize code of websocket close handler --- handler/real_time_data_pull.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/handler/real_time_data_pull.go b/handler/real_time_data_pull.go index c190ff1..5b298ae 100644 --- a/handler/real_time_data_pull.go +++ b/handler/real_time_data_pull.go @@ -60,6 +60,15 @@ func PullRealTimeDataHandler(c *gin.Context) { ctx, cancel := context.WithCancel(c.Request.Context()) defer cancel() + conn.SetCloseHandler(func(code int, text string) error { + logger.Info(c.Request.Context(), "websocket processor shutdown trigger", + "clientID", clientID, "code", code, "reason", text) + + // call cancel to notify other goroutines to stop working + cancel() + return nil + }) + // TODO[BACKPRESSURE-ISSUE] 先期使用固定大容量对扇入模型进行定义 #1 fanInChan := make(chan network.RealTimePullTarget, constants.FanInChanMaxSize) sendChan := make(chan []network.RealTimePullTarget, constants.SendChanBufferSize) From 1a1727adab9a44325a32ed41303eb5b9a78e2559 Mon Sep 17 00:00:00 2001 From: douxu Date: Mon, 26 Jan 2026 16:29:50 +0800 Subject: [PATCH 02/43] optimize reponse code and business code of measurement sub api --- common/errcode/bussiness_error.go | 6 + constants/attribute_business_code.go | 17 --- constants/business_code.go | 31 +++++ handler/real_time_data_subscription.go | 155 +++++++++---------------- network/response.go | 2 +- 5 files changed, 94 insertions(+), 117 deletions(-) delete mode 100644 constants/attribute_business_code.go create mode 100644 constants/business_code.go diff --git a/common/errcode/bussiness_error.go b/common/errcode/bussiness_error.go index 04ff3f6..08fcfea 100644 --- a/common/errcode/bussiness_error.go +++ b/common/errcode/bussiness_error.go @@ -16,6 +16,12 @@ var ( // ErrFoundTargetFailed define variable to returned when the specific database table cannot be identified using the provided token info. ErrFoundTargetFailed = newError(40004, "found target table by token failed") + // ErrSubTargetRepeat define variable to indicates subscription target already exist in list + ErrSubTargetRepeat = newError(40005, "subscription target already exist in list") + // ErrSubTargetNotFound define variable to indicates can not find measurement by subscription target + ErrSubTargetNotFound = newError(40006, "found measuremnet by subscription target failed") + // ErrCancelSubTargetMissing define variable to indicates cancel a not exist subscription target + ErrCancelSubTargetMissing = newError(40007, "cancel a not exist subscription target") // ErrDBQueryFailed define variable to represents a generic failure during a PostgreSQL SELECT or SCAN operation. ErrDBQueryFailed = newError(50001, "query postgres database data failed") diff --git a/constants/attribute_business_code.go b/constants/attribute_business_code.go deleted file mode 100644 index 487b338..0000000 --- a/constants/attribute_business_code.go +++ /dev/null @@ -1,17 +0,0 @@ -// Package constants define constant variable -package constants - -const ( - // CodeSuccess define constant to indicates that the API was successfully processed - CodeSuccess = 20000 - // CodeInvalidParamFailed define constant to indicates request parameter parsing failed - CodeInvalidParamFailed = 40001 - // CodeDBQueryFailed define constant to indicates database query operation failed - CodeDBQueryFailed = 50001 - // CodeDBUpdateailed define constant to indicates database update operation failed - CodeDBUpdateailed = 50002 - // CodeRedisQueryFailed define constant to indicates redis query operation failed - CodeRedisQueryFailed = 60001 - // CodeRedisUpdateFailed define constant to indicates redis update operation failed - CodeRedisUpdateFailed = 60002 -) diff --git a/constants/business_code.go b/constants/business_code.go new file mode 100644 index 0000000..aa18e0e --- /dev/null +++ b/constants/business_code.go @@ -0,0 +1,31 @@ +// Package constants define constant variable +package constants + +const ( + // CodeSuccess define constant to indicates that the API was successfully processed + CodeSuccess = 20000 + // CodeInvalidParamFailed define constant to indicates request parameter parsing failed + CodeInvalidParamFailed = 40001 + // CodeFoundTargetFailed define variable to returned when the specific database table cannot be identified using the provided token info. + CodeFoundTargetFailed = 40004 + // CodeSubTargetRepeat define variable to indicates subscription target already exist in list + CodeSubTargetRepeat = 40005 + // CodeSubTargetNotFound define variable to indicates can not find measurement by subscription target + CodeSubTargetNotFound = 40006 + // CodeCancelSubTargetMissing define variable to indicates cancel a not exist subscription target + CodeCancelSubTargetMissing = 40007 + // CodeUpdateSubTargetMissing define variable to indicates update a not exist subscription target + CodeUpdateSubTargetMissing = 40008 + // CodeAppendSubTargetMissing define variable to indicates append a not exist subscription target + CodeAppendSubTargetMissing = 40009 + // CodeUnsupportSubOperation define variable to indicates append a not exist subscription target + CodeUnsupportSubOperation = 40010 + // CodeDBQueryFailed define constant to indicates database query operation failed + CodeDBQueryFailed = 50001 + // CodeDBUpdateailed define constant to indicates database update operation failed + CodeDBUpdateailed = 50002 + // CodeRedisQueryFailed define constant to indicates redis query operation failed + CodeRedisQueryFailed = 60001 + // CodeRedisUpdateFailed define constant to indicates redis update operation failed + CodeRedisUpdateFailed = 60002 +) diff --git a/handler/real_time_data_subscription.go b/handler/real_time_data_subscription.go index f0f435b..4f87f33 100644 --- a/handler/real_time_data_subscription.go +++ b/handler/real_time_data_subscription.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "maps" - "net/http" "sync" "modelRT/constants" @@ -33,42 +32,42 @@ func init() { // @Accept json // @Produce json // @Param request body network.RealTimeSubRequest true "量测节点实时数据订阅" -// @Success 200 {object} network.SuccessResponse{payload=network.RealTimeSubPayload} "订阅实时数据结果列表" +// @Success 2000 {object} network.SuccessResponse{payload=network.RealTimeSubPayload} "订阅实时数据结果列表" // -// @Example 200 { -// "code": 200, -// "msg": "success", +// @Example 2000 { +// "code": 2000, +// "msg": "process completed", // "payload": { // "targets": [ // { // "id": "grid1.zone1.station1.ns1.tag1.bay.I11_C_rms", -// "code": "1001", +// "code": "20000", // "msg": "subscription success" // }, // { // "id": "grid1.zone1.station1.ns1.tag1.bay.I11_B_rms", -// "code": "1002", +// "code": "20000", // "msg": "subscription failed" // } // ] // } // } // -// @Failure 400 {object} network.FailureResponse{payload=network.RealTimeSubPayload} "订阅实时数据结果列表" +// @Failure 3000 {object} network.FailureResponse{payload=network.RealTimeSubPayload} "订阅实时数据结果列表" // -// @Example 400 { -// "code": 400, -// "msg": "failed to get recommend data from redis", +// @Example 3000 { +// "code": 3000, +// "msg": "process completed with partial failures", // "payload": { // "targets": [ // { // "id": "grid1.zone1.station1.ns1.tag1.bay.I11_A_rms", -// "code": "1002", +// "code": "40005", // "msg": "subscription failed" // }, // { // "id": "grid1.zone1.station1.ns1.tag1.bay.I11_B_rms", -// "code": "1002", +// "code": "50001", // "msg": "subscription failed" // } // ] @@ -83,10 +82,7 @@ func RealTimeSubHandler(c *gin.Context) { if err := c.ShouldBindJSON(&request); err != nil { logger.Error(c, "failed to unmarshal real time query request", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - }) + renderRespFailure(c, constants.RespCodeInvalidParams, err.Error(), nil) return } @@ -95,10 +91,7 @@ func RealTimeSubHandler(c *gin.Context) { id, err := uuid.NewV4() if err != nil { logger.Error(c, "failed to generate client id", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - }) + renderRespFailure(c, constants.RespCodeInvalidParams, err.Error(), nil) return } clientID = id.String() @@ -123,110 +116,74 @@ func RealTimeSubHandler(c *gin.Context) { results, err := globalSubState.CreateConfig(c, tx, clientID, request.Measurements) if err != nil { logger.Error(c, "create real time data subscription config failed", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespFailure(c, constants.RespCodeFailed, err.Error(), network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return } - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: http.StatusOK, - Msg: "success", - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespSuccess(c, constants.RespCodeSuccess, "process completed", network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return case constants.SubStopAction: results, err := globalSubState.RemoveTargets(c, clientID, request.Measurements) if err != nil { logger.Error(c, "remove target to real time data subscription config failed", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespFailure(c, constants.RespCodeFailed, err.Error(), network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return } - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: http.StatusOK, - Msg: "success", - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespSuccess(c, constants.RespCodeSuccess, "success", network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return case constants.SubAppendAction: results, err := globalSubState.AppendTargets(c, tx, clientID, request.Measurements) if err != nil { logger.Error(c, "append target to real time data subscription config failed", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespFailure(c, constants.RespCodeFailed, err.Error(), network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return } - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: http.StatusOK, - Msg: "success", - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespSuccess(c, constants.RespCodeSuccess, "success", network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return case constants.SubUpdateAction: results, err := globalSubState.UpdateTargets(c, tx, clientID, request.Measurements) if err != nil { logger.Error(c, "update target to real time data subscription config failed", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespFailure(c, constants.RespCodeFailed, err.Error(), network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return } - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: http.StatusOK, - Msg: "success", - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespSuccess(c, constants.RespCodeSuccess, "success", network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return default: err := fmt.Errorf("%w: request action is %s", constants.ErrUnsupportedSubAction, request.Action) logger.Error(c, "unsupported action of real time data subscription request", "error", err) requestTargetsCount := processRealTimeRequestCount(request.Measurements) - results := processRealTimeRequestTargets(request.Measurements, requestTargetsCount, err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + results := processRealTimeRequestTargets(request.Measurements, requestTargetsCount, constants.CodeUnsupportSubOperation, err) + renderRespFailure(c, constants.RespCodeInvalidParams, err.Error(), network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return } @@ -283,12 +240,12 @@ func processAndValidateTargetsForStart(ctx context.Context, tx *gorm.DB, measure targetModel, err := database.ParseDataIdentifierToken(ctx, tx, target) if err != nil { logger.Error(ctx, "parse data indentity token failed", "error", err, "identity_token", target) - targetResult.Code = constants.SubFailedCode + targetResult.Code = constants.CodeFoundTargetFailed targetResult.Msg = fmt.Sprintf("%s: %s", constants.SubFailedMsg, err.Error()) targetProcessResults = append(targetProcessResults, targetResult) continue } - targetResult.Code = constants.SubSuccessCode + targetResult.Code = constants.CodeSuccess targetResult.Msg = constants.SubSuccessMsg targetProcessResults = append(targetProcessResults, targetResult) successfulTargets = append(successfulTargets, target) @@ -327,7 +284,7 @@ func processAndValidateTargetsForUpdate(ctx context.Context, tx *gorm.DB, config if _, exist := config.targetContext[target]; !exist { err := fmt.Errorf("target %s does not exists in subscription list", target) logger.Error(ctx, "update target does not exist in subscription list", "error", err, "target", target) - targetResult.Code = constants.UpdateSubFailedCode + targetResult.Code = constants.CodeUpdateSubTargetMissing targetResult.Msg = fmt.Sprintf("%s: %s", constants.UpdateSubFailedMsg, err.Error()) targetProcessResults = append(targetProcessResults, targetResult) continue @@ -336,13 +293,13 @@ func processAndValidateTargetsForUpdate(ctx context.Context, tx *gorm.DB, config targetModel, err := database.ParseDataIdentifierToken(ctx, tx, target) if err != nil { logger.Error(ctx, "parse data indentity token failed", "error", err, "identity_token", target) - targetResult.Code = constants.UpdateSubFailedCode + targetResult.Code = constants.CodeDBQueryFailed targetResult.Msg = fmt.Sprintf("%s: %s", constants.UpdateSubFailedMsg, err.Error()) targetProcessResults = append(targetProcessResults, targetResult) continue } - targetResult.Code = constants.UpdateSubSuccessCode + targetResult.Code = constants.CodeSuccess targetResult.Msg = constants.UpdateSubSuccessMsg targetProcessResults = append(targetProcessResults, targetResult) successfulTargets = append(successfulTargets, target) @@ -473,7 +430,7 @@ func (s *SharedSubState) AppendTargets(ctx context.Context, tx *gorm.DB, clientI if !exist { err := fmt.Errorf("clientID %s not found. use CreateConfig to start a new config", clientID) logger.Error(ctx, "clientID not found. use CreateConfig to start a new config", "error", err) - return processRealTimeRequestTargets(measurements, requestTargetsCount, err), err + return processRealTimeRequestTargets(measurements, requestTargetsCount, constants.CodeAppendSubTargetMissing, err), err } targetProcessResults, successfulTargets, newMeasMap, newMeasContextMap := processAndValidateTargetsForStart(ctx, tx, measurements, requestTargetsCount) @@ -507,7 +464,7 @@ func filterAndDeduplicateRepeatTargets(resultsSlice []network.TargetResult, idsS for index := range resultsSlice { if _, isTarget := set[resultsSlice[index].ID]; isTarget { - resultsSlice[index].Code = constants.SubRepeatCode + resultsSlice[index].Code = constants.CodeSubTargetRepeat resultsSlice[index].Msg = constants.SubRepeatMsg } } @@ -575,7 +532,7 @@ func (s *SharedSubState) RemoveTargets(ctx context.Context, clientID string, mea s.globalMutex.RUnlock() err := fmt.Errorf("clientID %s not found", clientID) logger.Error(ctx, "clientID not found in remove targets operation", "error", err) - return processRealTimeRequestTargets(measurements, requestTargetsCount, err), err + return processRealTimeRequestTargets(measurements, requestTargetsCount, constants.CodeCancelSubTargetMissing, err), err } s.globalMutex.RUnlock() @@ -595,7 +552,7 @@ func (s *SharedSubState) RemoveTargets(ctx context.Context, clientID string, mea for _, target := range measTargets { targetResult := network.TargetResult{ ID: target, - Code: constants.CancelSubFailedCode, + Code: constants.CodeCancelSubTargetMissing, Msg: constants.CancelSubFailedMsg, } targetProcessResults = append(targetProcessResults, targetResult) @@ -616,7 +573,7 @@ func (s *SharedSubState) RemoveTargets(ctx context.Context, clientID string, mea transportTargets.Targets = append(transportTargets.Targets, existingTarget) targetResult := network.TargetResult{ ID: existingTarget, - Code: constants.CancelSubSuccessCode, + Code: constants.CodeSuccess, Msg: constants.CancelSubSuccessMsg, } targetProcessResults = append(targetProcessResults, targetResult) @@ -639,7 +596,7 @@ func (s *SharedSubState) RemoveTargets(ctx context.Context, clientID string, mea for target := range targetsToRemoveMap { targetResult := network.TargetResult{ ID: target, - Code: constants.CancelSubFailedCode, + Code: constants.CodeCancelSubTargetMissing, Msg: fmt.Sprintf("%s: %s", constants.SubFailedMsg, err.Error()), } targetProcessResults = append(targetProcessResults, targetResult) @@ -673,7 +630,7 @@ func (s *SharedSubState) UpdateTargets(ctx context.Context, tx *gorm.DB, clientI s.globalMutex.RUnlock() err := fmt.Errorf("clientID %s not found", clientID) logger.Error(ctx, "clientID not found in remove targets operation", "error", err) - return processRealTimeRequestTargets(measurements, requestTargetsCount, err), err + return processRealTimeRequestTargets(measurements, requestTargetsCount, constants.CodeUpdateSubTargetMissing, err), err } targetProcessResults, successfulTargets, newMeasMap, newMeasContextMap := processAndValidateTargetsForUpdate(ctx, tx, config, measurements, requestTargetsCount) @@ -722,13 +679,13 @@ func processRealTimeRequestCount(measurements []network.RealTimeMeasurementItem) return totalTargetsCount } -func processRealTimeRequestTargets(measurements []network.RealTimeMeasurementItem, targetCount int, err error) []network.TargetResult { +func processRealTimeRequestTargets(measurements []network.RealTimeMeasurementItem, targetCount int, businessCode int, err error) []network.TargetResult { targetProcessResults := make([]network.TargetResult, 0, targetCount) for _, measurementItem := range measurements { for _, target := range measurementItem.Targets { var targetResult network.TargetResult targetResult.ID = target - targetResult.Code = constants.SubFailedCode + targetResult.Code = businessCode targetResult.Msg = fmt.Sprintf("%s: %s", constants.SubFailedMsg, err.Error()) targetProcessResults = append(targetProcessResults, targetResult) } diff --git a/network/response.go b/network/response.go index e8d775a..4bed98d 100644 --- a/network/response.go +++ b/network/response.go @@ -26,7 +26,7 @@ type MeasurementRecommendPayload struct { // TargetResult define struct of target item in real time data subscription response payload type TargetResult struct { ID string `json:"id" example:"grid1.zone1.station1.ns1.tag1.transformfeeder1_220.I_A_rms"` - Code string `json:"code" example:"1001"` + Code int `json:"code" example:"20000"` Msg string `json:"msg" example:"subscription success"` } From 617d21500eab8bf4383d09cdc5e7be19539112f6 Mon Sep 17 00:00:00 2001 From: douxu Date: Tue, 27 Jan 2026 17:41:17 +0800 Subject: [PATCH 03/43] optimize code of redis connenct func and real time data calculate --- config/config.go | 12 ++-- ...usiness_code.go => rtdata_subscription.go} | 23 -------- diagram/redis_client.go | 4 +- diagram/redis_init.go | 4 +- distributedlock/locker_init.go | 4 +- handler/real_time_data_pull.go | 2 +- handler/real_time_data_subscription.go | 1 - real-time-data/real_time_data_computing.go | 9 ++- util/redis_init.go | 58 +++++++++++++------ util/redis_options.go | 55 ++++++++++++------ 10 files changed, 101 insertions(+), 71 deletions(-) rename constants/{subscription_business_code.go => rtdata_subscription.go} (74%) diff --git a/config/config.go b/config/config.go index 21e4129..f44c11b 100644 --- a/config/config.go +++ b/config/config.go @@ -53,11 +53,13 @@ type LoggerConfig struct { // RedisConfig define config struct of redis config type RedisConfig struct { - Addr string `mapstructure:"addr"` - Password string `mapstructure:"password"` - DB int `mapstructure:"db"` - PoolSize int `mapstructure:"poolsize"` - Timeout int `mapstructure:"timeout"` + Addr string `mapstructure:"addr"` + Password string `mapstructure:"password"` + DB int `mapstructure:"db"` + PoolSize int `mapstructure:"poolsize"` + DialTimeout int `mapstructure:"dial_timeout"` + ReadTimeout int `mapstructure:"read_timeout"` + WriteTimeout int `mapstructure:"write_timeout"` } // AntsConfig define config struct of ants pool config diff --git a/constants/subscription_business_code.go b/constants/rtdata_subscription.go similarity index 74% rename from constants/subscription_business_code.go rename to constants/rtdata_subscription.go index 4b202bb..e14a3da 100644 --- a/constants/subscription_business_code.go +++ b/constants/rtdata_subscription.go @@ -12,29 +12,6 @@ const ( SubUpdateAction string = "update" ) -// 定义状态常量 -// TODO 从4位格式修改为5位格式 -const ( - // SubSuccessCode define subscription success code - SubSuccessCode = "1001" - // SubFailedCode define subscription failed code - SubFailedCode = "1002" - // RTDSuccessCode define real time data return success code - RTDSuccessCode = "1003" - // RTDFailedCode define real time data return failed code - RTDFailedCode = "1004" - // CancelSubSuccessCode define cancel subscription success code - CancelSubSuccessCode = "1005" - // CancelSubFailedCode define cancel subscription failed code - CancelSubFailedCode = "1006" - // SubRepeatCode define subscription repeat code - SubRepeatCode = "1007" - // UpdateSubSuccessCode define update subscription success code - UpdateSubSuccessCode = "1008" - // UpdateSubFailedCode define update subscription failed code - UpdateSubFailedCode = "1009" -) - const ( // SysCtrlPrefix define to indicates the prefix for all system control directives,facilitating unified parsing within the sendDataStream goroutine SysCtrlPrefix = "SYS_CTRL_" diff --git a/diagram/redis_client.go b/diagram/redis_client.go index 283669d..4b673a8 100644 --- a/diagram/redis_client.go +++ b/diagram/redis_client.go @@ -19,8 +19,8 @@ func NewRedisClient() *RedisClient { } } -// QueryByZRangeByLex define func to query real time data from redis zset -func (rc *RedisClient) QueryByZRangeByLex(ctx context.Context, key string, size int64) ([]redis.Z, error) { +// QueryByZRange define func to query real time data from redis zset +func (rc *RedisClient) QueryByZRange(ctx context.Context, key string, size int64) ([]redis.Z, error) { client := rc.Client args := redis.ZRangeArgs{ Key: key, diff --git a/diagram/redis_init.go b/diagram/redis_init.go index d1d7a22..273dd22 100644 --- a/diagram/redis_init.go +++ b/diagram/redis_init.go @@ -22,7 +22,9 @@ func initClient(rCfg config.RedisConfig) *redis.Client { util.WithPassword(rCfg.Password), util.WithDB(rCfg.DB), util.WithPoolSize(rCfg.PoolSize), - util.WithTimeout(time.Duration(rCfg.Timeout)*time.Second), + util.WithConnectTimeout(time.Duration(rCfg.DialTimeout)*time.Second), + util.WithReadTimeout(time.Duration(rCfg.ReadTimeout)*time.Second), + util.WithWriteTimeout(time.Duration(rCfg.WriteTimeout)*time.Second), ) if err != nil { panic(err) diff --git a/distributedlock/locker_init.go b/distributedlock/locker_init.go index 35ecc78..e3a7bc2 100644 --- a/distributedlock/locker_init.go +++ b/distributedlock/locker_init.go @@ -22,7 +22,9 @@ func initClient(rCfg config.RedisConfig) *redis.Client { util.WithPassword(rCfg.Password), util.WithDB(rCfg.DB), util.WithPoolSize(rCfg.PoolSize), - util.WithTimeout(time.Duration(rCfg.Timeout)*time.Second), + util.WithConnectTimeout(time.Duration(rCfg.DialTimeout)*time.Second), + util.WithReadTimeout(time.Duration(rCfg.ReadTimeout)*time.Second), + util.WithWriteTimeout(time.Duration(rCfg.WriteTimeout)*time.Second), ) if err != nil { panic(err) diff --git a/handler/real_time_data_pull.go b/handler/real_time_data_pull.go index 5b298ae..a360a49 100644 --- a/handler/real_time_data_pull.go +++ b/handler/real_time_data_pull.go @@ -472,7 +472,7 @@ func realTimeDataQueryFromRedis(ctx context.Context, config redisPollingConfig, } func performQuery(ctx context.Context, client *diagram.RedisClient, config redisPollingConfig, fanInChan chan network.RealTimePullTarget) { - members, err := client.QueryByZRangeByLex(ctx, config.queryKey, config.dataSize) + members, err := client.QueryByZRange(ctx, config.queryKey, config.dataSize) if err != nil { logger.Error(ctx, "query real time data from redis failed", "key", config.queryKey, "error", err) return diff --git a/handler/real_time_data_subscription.go b/handler/real_time_data_subscription.go index 4f87f33..afc396b 100644 --- a/handler/real_time_data_subscription.go +++ b/handler/real_time_data_subscription.go @@ -627,7 +627,6 @@ func (s *SharedSubState) UpdateTargets(ctx context.Context, tx *gorm.DB, clientI s.globalMutex.RUnlock() if !exist { - s.globalMutex.RUnlock() err := fmt.Errorf("clientID %s not found", clientID) logger.Error(ctx, "clientID not found in remove targets operation", "error", err) return processRealTimeRequestTargets(measurements, requestTargetsCount, constants.CodeUpdateSubTargetMissing, err), err diff --git a/real-time-data/real_time_data_computing.go b/real-time-data/real_time_data_computing.go index 6de1ab1..88a2a8a 100644 --- a/real-time-data/real_time_data_computing.go +++ b/real-time-data/real_time_data_computing.go @@ -205,13 +205,20 @@ func continuousComputation(ctx context.Context, conf *ComputeConfig) { logger.Info(ctx, "continuous computing goroutine stopped by parent context done signal") return case <-ticker.C: - members, err := client.QueryByZRangeByLex(ctx, conf.QueryKey, conf.DataSize) + queryCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + members, err := client.QueryByZRange(queryCtx, conf.QueryKey, conf.DataSize) + cancel() if err != nil { logger.Error(ctx, "query real time data from redis failed", "key", conf.QueryKey, "error", err) continue } realTimedatas := util.ConvertZSetMembersToFloat64(members) + if len(realTimedatas) == 0 { + logger.Info(ctx, "no real time data queried from redis, skip this computation cycle", "key", conf.QueryKey) + continue + } + if conf.Analyzer != nil { conf.Analyzer.AnalyzeAndTriggerEvent(ctx, conf, realTimedatas) } else { diff --git a/util/redis_init.go b/util/redis_init.go index 8ea504a..f2c61a2 100644 --- a/util/redis_init.go +++ b/util/redis_init.go @@ -3,6 +3,7 @@ package util import ( "context" + "errors" "fmt" "time" @@ -13,25 +14,36 @@ import ( ) // NewRedisClient define func of initialize the Redis client -func NewRedisClient(addr string, opts ...RedisOption) (*redis.Client, error) { +func NewRedisClient(addr string, opts ...Option) (*redis.Client, error) { // default options - options := RedisOptions{ - redisOptions: &redis.Options{ - Addr: addr, + configs := &clientConfig{ + Options: &redis.Options{ + Addr: addr, + DialTimeout: 5 * time.Second, + ReadTimeout: 3 * time.Second, + WriteTimeout: 3 * time.Second, + PoolSize: 10, }, } // Apply configuration options from config + var errs []error for _, opt := range opts { - opt(&options) + if err := opt(configs); err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return nil, fmt.Errorf("failed to apply options: %w", errors.Join(errs...)) } // create redis client - client := redis.NewClient(options.redisOptions) + client := redis.NewClient(configs.Options) - if options.timeout > 0 { + if configs.DialTimeout > 0 { // check if the connection is successful - ctx, cancel := context.WithTimeout(context.Background(), options.timeout) + ctx, cancel := context.WithTimeout(context.Background(), configs.DialTimeout) defer cancel() if err := client.Ping(ctx).Err(); err != nil { return nil, fmt.Errorf("can not connect redis:%v", err) @@ -43,22 +55,29 @@ func NewRedisClient(addr string, opts ...RedisOption) (*redis.Client, error) { // NewRedigoPool define func of initialize the Redigo pool func NewRedigoPool(rCfg config.RedisConfig) (*redigo.Pool, error) { pool := &redigo.Pool{ - MaxIdle: rCfg.PoolSize / 2, - MaxActive: rCfg.PoolSize, // TODO optimize IdleTimeout with config parameter + MaxIdle: rCfg.PoolSize / 2, + MaxActive: rCfg.PoolSize, IdleTimeout: 240 * time.Second, + TestOnBorrow: func(c redigo.Conn, t time.Time) error { + if time.Since(t) < time.Minute { + return nil + } + _, err := c.Do("PING") + return err + }, - // Dial function to create the connection Dial: func() (redigo.Conn, error) { - timeout := time.Duration(rCfg.Timeout) * time.Millisecond // 假设 rCfg.Timeout 是毫秒 + dialTimeout := time.Duration(rCfg.DialTimeout) * time.Second + readTimeout := time.Duration(rCfg.ReadTimeout) * time.Second + writeTimeout := time.Duration(rCfg.WriteTimeout) * time.Second + opts := []redigo.DialOption{ redigo.DialDatabase(rCfg.DB), redigo.DialPassword(rCfg.Password), - redigo.DialConnectTimeout(timeout), - // redigo.DialReadTimeout(timeout), - // redigo.DialWriteTimeout(timeout), - // TODO add redigo.DialUsername when redis open acl - // redis.DialUsername("username"), + redigo.DialConnectTimeout(dialTimeout), + redigo.DialReadTimeout(readTimeout), + redigo.DialWriteTimeout(writeTimeout), } c, err := redigo.Dial("tcp", rCfg.Addr, opts...) @@ -72,13 +91,14 @@ func NewRedigoPool(rCfg config.RedisConfig) (*redigo.Pool, error) { conn := pool.Get() defer conn.Close() - if conn.Err() != nil { - return nil, fmt.Errorf("failed to get connection from pool: %w", conn.Err()) + if err := conn.Err(); err != nil { + return nil, fmt.Errorf("failed to get connection from pool: %w", err) } _, err := conn.Do("PING") if err != nil { return nil, fmt.Errorf("redis connection test (PING) failed: %w", err) } + return pool, nil } diff --git a/util/redis_options.go b/util/redis_options.go index dbb4feb..6293105 100644 --- a/util/redis_options.go +++ b/util/redis_options.go @@ -8,53 +8,74 @@ import ( "github.com/redis/go-redis/v9" ) -type RedisOptions struct { - redisOptions *redis.Options - timeout time.Duration +type clientConfig struct { + *redis.Options } -type RedisOption func(*RedisOptions) error +type Option func(*clientConfig) error // WithPassword define func of configure redis password options -func WithPassword(password string) RedisOption { - return func(o *RedisOptions) error { +func WithPassword(password string) Option { + return func(c *clientConfig) error { if password == "" { return errors.New("password is empty") } - o.redisOptions.Password = password + c.Password = password return nil } } -// WithTimeout define func of configure redis timeout options -func WithTimeout(timeout time.Duration) RedisOption { - return func(o *RedisOptions) error { +// WithConnectTimeout define func of configure redis connect timeout options +func WithConnectTimeout(timeout time.Duration) Option { + return func(c *clientConfig) error { if timeout < 0 { return errors.New("timeout can not be negative") } - o.timeout = timeout + c.DialTimeout = timeout + return nil + } +} + +// WithReadTimeout define func of configure redis read timeout options +func WithReadTimeout(timeout time.Duration) Option { + return func(c *clientConfig) error { + if timeout < 0 { + return errors.New("timeout can not be negative") + } + c.ReadTimeout = timeout + return nil + } +} + +// WithWriteTimeout define func of configure redis write timeout options +func WithWriteTimeout(timeout time.Duration) Option { + return func(c *clientConfig) error { + if timeout < 0 { + return errors.New("timeout can not be negative") + } + c.WriteTimeout = timeout return nil } } // WithDB define func of configure redis db options -func WithDB(db int) RedisOption { - return func(o *RedisOptions) error { +func WithDB(db int) Option { + return func(c *clientConfig) error { if db < 0 { return errors.New("db can not be negative") } - o.redisOptions.DB = db + c.DB = db return nil } } // WithPoolSize define func of configure pool size options -func WithPoolSize(poolSize int) RedisOption { - return func(o *RedisOptions) error { +func WithPoolSize(poolSize int) Option { + return func(c *clientConfig) error { if poolSize <= 0 { return errors.New("pool size must be greater than 0") } - o.redisOptions.PoolSize = poolSize + c.PoolSize = poolSize return nil } } From 3ff29cc0727bc723249fc619735522ecdd54dd58 Mon Sep 17 00:00:00 2001 From: douxu Date: Wed, 28 Jan 2026 14:03:25 +0800 Subject: [PATCH 04/43] optimize code of real time data pull api --- handler/helper.go | 11 +++ handler/real_time_data_pull.go | 127 +++++++++++++-------------------- network/response.go | 15 ++-- 3 files changed, 70 insertions(+), 83 deletions(-) diff --git a/handler/helper.go b/handler/helper.go index 6589409..532d93b 100644 --- a/handler/helper.go +++ b/handler/helper.go @@ -30,3 +30,14 @@ func renderRespSuccess(c *gin.Context, code int, msg string, payload any) { } c.JSON(http.StatusOK, resp) } + +func renderWSRespFailure(c *gin.Context, code int, msg string, payload any) { + resp := network.WSResponse{ + Code: code, + Msg: msg, + } + if payload != nil { + resp.Payload = payload + } + c.JSON(http.StatusOK, resp) +} diff --git a/handler/real_time_data_pull.go b/handler/real_time_data_pull.go index a360a49..24ccf50 100644 --- a/handler/real_time_data_pull.go +++ b/handler/real_time_data_pull.go @@ -39,20 +39,14 @@ func PullRealTimeDataHandler(c *gin.Context) { if clientID == "" { err := fmt.Errorf("clientID is missing from the path") logger.Error(c, "query clientID from path failed", "error", err, "url", c.Request.RequestURI) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - }) + renderWSRespFailure(c, constants.RespCodeInvalidParams, err.Error(), nil) return } conn, err := pullUpgrader.Upgrade(c.Writer, c.Request, nil) if err != nil { - logger.Error(c, "upgrade http protocol to websocket protocal failed", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - }) + logger.Error(c, "upgrade http protocol to websocket protocol failed", "error", err) + renderWSRespFailure(c, constants.RespCodeServerError, err.Error(), nil) return } defer conn.Close() @@ -71,7 +65,7 @@ func PullRealTimeDataHandler(c *gin.Context) { // TODO[BACKPRESSURE-ISSUE] 先期使用固定大容量对扇入模型进行定义 #1 fanInChan := make(chan network.RealTimePullTarget, constants.FanInChanMaxSize) - sendChan := make(chan []network.RealTimePullTarget, constants.SendChanBufferSize) + sendChan := make(chan network.WSResponse, constants.SendChanBufferSize) go processTargetPolling(ctx, globalSubState, clientID, fanInChan, sendChan) go readClientMessages(ctx, conn, clientID, cancel) @@ -88,52 +82,33 @@ func PullRealTimeDataHandler(c *gin.Context) { select { case targetData, ok := <-fanInChan: if !ok { - logger.Error(ctx, "fanInChan closed unexpectedly", "client_id", clientID) + sendChan <- network.WSResponse{ + Code: constants.RespCodeServerError, + Msg: "abnormal shutdown of data fan-in channel", + } return } + buffer = append(buffer, targetData) - if len(buffer) >= bufferMaxSize { - // buffer is full, send immediately - select { - case sendChan <- buffer: - default: - logger.Warn(ctx, "sendChan is full, dropping aggregated data batch (buffer is full)", "client_id", clientID) - } - - // reset buffer - buffer = make([]network.RealTimePullTarget, 0, bufferMaxSize) - // reset the ticker to prevent it from triggering immediately after the ticker is sent + flushBuffer(ctx, &buffer, sendChan, clientID, "buffer_full") ticker.Reset(sendMaxInterval) } case <-ticker.C: if len(buffer) > 0 { - // when the ticker is triggered, all data in the send buffer is sent - select { - case sendChan <- buffer: - default: - logger.Warn(ctx, "sendChan is full, dropping aggregated data batch (ticker is triggered)", "client_id", clientID) - } - - // reset buffer - buffer = make([]network.RealTimePullTarget, 0, bufferMaxSize) + flushBuffer(ctx, &buffer, sendChan, clientID, "ticker_timeout") } case <-ctx.Done(): - // send the last remaining data + // last refresh before exiting if len(buffer) > 0 { - select { - case sendChan <- buffer: - default: - logger.Warn(ctx, "sendChan is full, cannot send last remaining data during shutdown.", "client_id", clientID) - } + flushBuffer(ctx, &buffer, sendChan, clientID, "shutdown") } - logger.Info(ctx, "pullRealTimeDataHandler exiting as context is done.", "client_id", clientID) return } } } -// readClientMessages 负责持续监听客户端发送的消息（例如 Ping/Pong, Close Frame, 或控制命令） +// readClientMessages define func to responsible for continuously listening for messages sent by clients (such as Ping/Pong, Close Frame, or control commands) func readClientMessages(ctx context.Context, conn *websocket.Conn, clientID string, cancel context.CancelFunc) { // conn.SetReadLimit(512) for { @@ -158,54 +133,47 @@ func readClientMessages(ctx context.Context, conn *websocket.Conn, clientID stri } } -// sendAggregateRealTimeDataStream define func to responsible for continuously pushing aggregate real-time data to the client -func sendAggregateRealTimeDataStream(conn *websocket.Conn, targetsData []network.RealTimePullTarget) error { - if len(targetsData) == 0 { - return nil +func flushBuffer(ctx context.Context, buffer *[]network.RealTimePullTarget, sendChan chan<- network.WSResponse, clientID string, reason string) { + if len(*buffer) == 0 { + return } - response := network.SuccessResponse{ - Code: 200, - Msg: "success", + + resp := network.WSResponse{ + Code: constants.RespCodeSuccess, + Msg: "process completed", Payload: network.RealTimePullPayload{ - Targets: targetsData, + Targets: *buffer, }, } - return conn.WriteJSON(response) + + select { + case sendChan <- resp: + default: + logger.Warn(ctx, "sendChan blocked, dropping data batch", "client_id", clientID, "reason", reason) + } + *buffer = make([]network.RealTimePullTarget, 0, constants.SendMaxBatchSize) } // sendDataStream define func to manages a dedicated goroutine to push data batches or system signals to the websocket client -func sendDataStream(ctx context.Context, conn *websocket.Conn, clientID string, sendChan <-chan []network.RealTimePullTarget, cancel context.CancelFunc) { - logger.Info(ctx, "start dedicated websocket sender goroutine", "client_id", clientID) - for targetsData := range sendChan { - // TODO 使用 constants.SysCtrlPrefix + switch-case 形式应对可能的业务扩展 - if len(targetsData) == 1 && targetsData[0].ID == constants.SysCtrlAllRemoved { - err := conn.WriteJSON(map[string]any{ - "code": 2101, - "msg": "all targets removed in given client_id", - "payload": map[string]int{ - "active_targets_count": 0, - }, - }) - if err != nil { - logger.Error(ctx, "send all targets removed system signal failed", "client_id", clientID, "error", err) - cancel() - } - continue +func sendDataStream(ctx context.Context, conn *websocket.Conn, clientID string, sendChan <-chan network.WSResponse, cancel context.CancelFunc) { + defer func() { + if r := recover(); r != nil { + logger.Error(ctx, "sendDataStream recovered from panic", "err", r) } + }() - if err := sendAggregateRealTimeDataStream(conn, targetsData); err != nil { - logger.Error(ctx, "send the real time aggregate data failed in sender goroutine", "client_id", clientID, "error", err) + logger.Info(ctx, "start dedicated websocket sender goroutine", "client_id", clientID) + for resp := range sendChan { + if err := conn.WriteJSON(resp); err != nil { + logger.Error(ctx, "websocket write failed", "client_id", clientID, "error", err) cancel() return } } - logger.Info(ctx, "sender goroutine exiting as channel is closed", "client_id", clientID) } // processTargetPolling define func to process target in subscription map and data is continuously retrieved from redis based on the target -func processTargetPolling(ctx context.Context, s *SharedSubState, clientID string, fanInChan chan network.RealTimePullTarget, sendChan chan<- []network.RealTimePullTarget) { - // ensure the fanInChan will not leak - defer close(fanInChan) +func processTargetPolling(ctx context.Context, s *SharedSubState, clientID string, fanInChan chan network.RealTimePullTarget, sendChan chan<- network.WSResponse) { logger.Info(ctx, fmt.Sprintf("start processing real time data polling for clientID:%s", clientID)) stopChanMap := make(map[string]chan struct{}) s.globalMutex.RLock() @@ -392,7 +360,7 @@ func updateTargets(ctx context.Context, config *RealTimeSubConfig, stopChanMap m } // removeTargets define func to stops running polling goroutines for targets that were removed -func removeTargets(ctx context.Context, stopChanMap map[string]chan struct{}, removeTargets []string, sendChan chan<- []network.RealTimePullTarget) { +func removeTargets(ctx context.Context, stopChanMap map[string]chan struct{}, removeTargets []string, sendChan chan<- network.WSResponse) { for _, target := range removeTargets { stopChan, exists := stopChanMap[target] if !exists { @@ -411,17 +379,18 @@ func removeTargets(ctx context.Context, stopChanMap map[string]chan struct{}, re } } -func sendSpecialStatusToClient(ctx context.Context, sendChan chan<- []network.RealTimePullTarget) { - specialTarget := network.RealTimePullTarget{ - ID: constants.SysCtrlAllRemoved, - Datas: []network.RealTimePullData{}, +func sendSpecialStatusToClient(ctx context.Context, sendChan chan<- network.WSResponse) { + // TODO 使用 constants.SysCtrlPrefix + switch-case 形式应对可能的业务扩展 + resp := network.WSResponse{ + Code: constants.RespCodeSuccessWithNoSub, + Msg: "all targets removed", + Payload: map[string]int{"active_targets_count": 0}, } select { - case sendChan <- []network.RealTimePullTarget{specialTarget}: - logger.Info(ctx, "sent 2101 status request to sendChan") + case sendChan <- resp: default: - logger.Warn(ctx, "sendChan is full, skipping 2101 status message") + logger.Warn(ctx, "sendChan is full, skipping 2101 status") } } diff --git a/network/response.go b/network/response.go index 4bed98d..b111df7 100644 --- a/network/response.go +++ b/network/response.go @@ -3,18 +3,25 @@ package network // FailureResponse define struct of standard failure API response format type FailureResponse struct { - Code int `json:"code" example:"500"` - Msg string `json:"msg" example:"failed to get recommend data from redis"` + Code int `json:"code" example:"3000"` + Msg string `json:"msg" example:"process completed with partial failures"` Payload any `json:"payload" swaggertype:"object"` } // SuccessResponse define struct of standard successful API response format type SuccessResponse struct { - Code int `json:"code" example:"200"` - Msg string `json:"msg" example:"success"` + Code int `json:"code" example:"2000"` + Msg string `json:"msg" example:"process completed"` Payload any `json:"payload" swaggertype:"object"` } +// WSResponse define struct of standard websocket API response format +type WSResponse struct { + Code int `json:"code" example:"2000"` + Msg string `json:"msg" example:"process completed"` + Payload any `json:"payload,omitempty" swaggertype:"object"` +} + // MeasurementRecommendPayload define struct of represents the data payload for the successful recommendation response. type MeasurementRecommendPayload struct { Input string `json:"input" example:"transformfeeder1_220."` From 3374eec047030d5ee9e29a70a17ed091789eaa20 Mon Sep 17 00:00:00 2001 From: douxu Date: Wed, 28 Jan 2026 16:49:12 +0800 Subject: [PATCH 05/43] optimize code of redis init --- config/config.go | 1 + constants/deploy_mode.go | 11 +++++++++++ diagram/redis_init.go | 8 ++++---- distributedlock/locker_init.go | 8 ++++---- main.go | 4 ++-- util/redis_options.go | 6 ++++-- 6 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 constants/deploy_mode.go diff --git a/config/config.go b/config/config.go index f44c11b..b6379d3 100644 --- a/config/config.go +++ b/config/config.go @@ -19,6 +19,7 @@ type ServiceConfig struct { ServiceAddr string `mapstructure:"service_addr"` ServiceName string `mapstructure:"service_name"` SecretKey string `mapstructure:"secret_key"` + DeployEnv string `mapstructure:"deploy_env"` } // KafkaConfig define config struct of kafka config diff --git a/constants/deploy_mode.go b/constants/deploy_mode.go new file mode 100644 index 0000000..2a5addf --- /dev/null +++ b/constants/deploy_mode.go @@ -0,0 +1,11 @@ +// Package constants define constant variable +package constants + +const ( + // DevelopmentDeployMode define development operator environment for modelRT project + DevelopmentDeployMode = "development" + // DebugDeployMode define debug operator environment for modelRT project + DebugDeployMode = "debug" + // ProductionDeployMode define production operator environment for modelRT project + ProductionDeployMode = "production" +) diff --git a/diagram/redis_init.go b/diagram/redis_init.go index 273dd22..639f9f0 100644 --- a/diagram/redis_init.go +++ b/diagram/redis_init.go @@ -16,10 +16,10 @@ var ( ) // initClient define func of return successfully initialized redis client -func initClient(rCfg config.RedisConfig) *redis.Client { +func initClient(rCfg config.RedisConfig, deployEnv string) *redis.Client { client, err := util.NewRedisClient( rCfg.Addr, - util.WithPassword(rCfg.Password), + util.WithPassword(rCfg.Password, deployEnv), util.WithDB(rCfg.DB), util.WithPoolSize(rCfg.PoolSize), util.WithConnectTimeout(time.Duration(rCfg.DialTimeout)*time.Second), @@ -33,9 +33,9 @@ func initClient(rCfg config.RedisConfig) *redis.Client { } // InitRedisClientInstance define func of return instance of redis client -func InitRedisClientInstance(rCfg config.RedisConfig) *redis.Client { +func InitRedisClientInstance(rCfg config.RedisConfig, deployEnv string) *redis.Client { once.Do(func() { - _globalStorageClient = initClient(rCfg) + _globalStorageClient = initClient(rCfg, deployEnv) }) return _globalStorageClient } diff --git a/distributedlock/locker_init.go b/distributedlock/locker_init.go index e3a7bc2..507b00f 100644 --- a/distributedlock/locker_init.go +++ b/distributedlock/locker_init.go @@ -16,10 +16,10 @@ var ( ) // initClient define func of return successfully initialized redis client -func initClient(rCfg config.RedisConfig) *redis.Client { +func initClient(rCfg config.RedisConfig, deployEnv string) *redis.Client { client, err := util.NewRedisClient( rCfg.Addr, - util.WithPassword(rCfg.Password), + util.WithPassword(rCfg.Password, deployEnv), util.WithDB(rCfg.DB), util.WithPoolSize(rCfg.PoolSize), util.WithConnectTimeout(time.Duration(rCfg.DialTimeout)*time.Second), @@ -33,9 +33,9 @@ func initClient(rCfg config.RedisConfig) *redis.Client { } // InitClientInstance define func of return instance of redis client -func InitClientInstance(rCfg config.RedisConfig) *redis.Client { +func InitClientInstance(rCfg config.RedisConfig, deployEnv string) *redis.Client { once.Do(func() { - _globalLockerClient = initClient(rCfg) + _globalLockerClient = initClient(rCfg, deployEnv) }) return _globalLockerClient } diff --git a/main.go b/main.go index a875a74..bd06afe 100644 --- a/main.go +++ b/main.go @@ -130,10 +130,10 @@ func main() { defer searchPool.Close() model.InitAutocompleterWithPool(searchPool) - storageClient := diagram.InitRedisClientInstance(modelRTConfig.StorageRedisConfig) + storageClient := diagram.InitRedisClientInstance(modelRTConfig.StorageRedisConfig, *&modelRTConfig.ServiceConfig.DeployEnv) defer storageClient.Close() - lockerClient := locker.InitClientInstance(modelRTConfig.LockerRedisConfig) + lockerClient := locker.InitClientInstance(modelRTConfig.LockerRedisConfig, *&modelRTConfig.ServiceConfig.DeployEnv) defer lockerClient.Close() // init anchor param ants pool diff --git a/util/redis_options.go b/util/redis_options.go index 6293105..6980ecd 100644 --- a/util/redis_options.go +++ b/util/redis_options.go @@ -5,6 +5,8 @@ import ( "errors" "time" + "modelRT/constants" + "github.com/redis/go-redis/v9" ) @@ -15,9 +17,9 @@ type clientConfig struct { type Option func(*clientConfig) error // WithPassword define func of configure redis password options -func WithPassword(password string) Option { +func WithPassword(password string, env string) Option { return func(c *clientConfig) error { - if password == "" { + if env == constants.ProductionDeployMode && password == "" { return errors.New("password is empty") } c.Password = password From 2126aa7b069e087f6be36fc037d8913689a99046 Mon Sep 17 00:00:00 2001 From: douxu Date: Thu, 29 Jan 2026 17:00:20 +0800 Subject: [PATCH 06/43] optimize code of config --- config/anchor_param_config.go | 5 +++-- database/fill_identity_token_model.go | 5 +++-- database/filling_attr_model_info.go | 5 +++-- database/postgres_init.go | 10 +++------- main.go | 21 ++++++++++++++------- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/config/anchor_param_config.go b/config/anchor_param_config.go index 018aca9..bd9d7f3 100644 --- a/config/anchor_param_config.go +++ b/config/anchor_param_config.go @@ -44,10 +44,11 @@ var baseCurrentFunc = func(archorValue float64, args ...float64) float64 { // SelectAnchorCalculateFuncAndParams define select anchor func and anchor calculate value by component type 、 anchor name and component data func SelectAnchorCalculateFuncAndParams(componentType int, anchorName string, componentData map[string]interface{}) (func(archorValue float64, args ...float64) float64, []float64) { if componentType == constants.DemoType { - if anchorName == "voltage" { + switch anchorName { + case "voltage": resistance := componentData["resistance"].(float64) return baseVoltageFunc, []float64{resistance} - } else if anchorName == "current" { + case "current": resistance := componentData["resistance"].(float64) return baseCurrentFunc, []float64{resistance} } diff --git a/database/fill_identity_token_model.go b/database/fill_identity_token_model.go index 78a0e48..5f0fdd5 100644 --- a/database/fill_identity_token_model.go +++ b/database/fill_identity_token_model.go @@ -53,7 +53,8 @@ func FillingLongTokenModel(ctx context.Context, tx *gorm.DB, identModel *model.L func ParseDataIdentifierToken(ctx context.Context, tx *gorm.DB, identToken string) (model.IndentityTokenModelInterface, error) { identSlice := strings.Split(identToken, ".") identSliceLen := len(identSlice) - if identSliceLen == 4 { + switch identSliceLen { + case 4: // token1.token2.token3.token4.token7 shortIndentModel := &model.ShortIdentityTokenModel{ GridTag: identSlice[0], @@ -67,7 +68,7 @@ func ParseDataIdentifierToken(ctx context.Context, tx *gorm.DB, identToken strin return nil, err } return shortIndentModel, nil - } else if identSliceLen == 7 { + case 7: // token1.token2.token3.token4.token5.token6.token7 longIndentModel := &model.LongIdentityTokenModel{ GridTag: identSlice[0], diff --git a/database/filling_attr_model_info.go b/database/filling_attr_model_info.go index 408f7f7..6debf21 100644 --- a/database/filling_attr_model_info.go +++ b/database/filling_attr_model_info.go @@ -19,7 +19,8 @@ func ParseAttrToken(ctx context.Context, tx *gorm.DB, attrToken, clientToken str attrSlice := strings.Split(attrToken, ".") attrLen := len(attrSlice) - if attrLen == 4 { + switch attrLen { + case 4: short := &model.ShortAttrInfo{ AttrGroupName: attrSlice[2], AttrKey: attrSlice[3], @@ -35,7 +36,7 @@ func ParseAttrToken(ctx context.Context, tx *gorm.DB, attrToken, clientToken str } short.AttrValue = attrValue return short, nil - } else if attrLen == 7 { + case 7: long := &model.LongAttrInfo{ AttrGroupName: attrSlice[5], AttrKey: attrSlice[6], diff --git a/database/postgres_init.go b/database/postgres_init.go index a80d9fc..bd7869f 100644 --- a/database/postgres_init.go +++ b/database/postgres_init.go @@ -2,9 +2,7 @@ package database import ( - "context" "sync" - "time" "modelRT/logger" @@ -27,17 +25,15 @@ func GetPostgresDBClient() *gorm.DB { } // InitPostgresDBInstance return instance of PostgresDB client -func InitPostgresDBInstance(ctx context.Context, PostgresDBURI string) *gorm.DB { +func InitPostgresDBInstance(PostgresDBURI string) *gorm.DB { postgresOnce.Do(func() { - _globalPostgresClient = initPostgresDBClient(ctx, PostgresDBURI) + _globalPostgresClient = initPostgresDBClient(PostgresDBURI) }) return _globalPostgresClient } // initPostgresDBClient return successfully initialized PostgresDB client -func initPostgresDBClient(ctx context.Context, PostgresDBURI string) *gorm.DB { - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() +func initPostgresDBClient(PostgresDBURI string) *gorm.DB { db, err := gorm.Open(postgres.Open(PostgresDBURI), &gorm.Config{Logger: logger.NewGormLogger()}) if err != nil { panic(err) diff --git a/main.go b/main.go index bd06afe..a0aca1e 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ import ( "modelRT/alert" "modelRT/config" + "modelRT/constants" "modelRT/database" "modelRT/diagram" "modelRT/logger" @@ -98,14 +99,14 @@ func main() { panic(err) } - serviceToken, err := util.GenerateClientToken(hostName, modelRTConfig.ServiceConfig.ServiceName, modelRTConfig.ServiceConfig.SecretKey) + serviceToken, err := util.GenerateClientToken(hostName, modelRTConfig.ServiceName, modelRTConfig.SecretKey) if err != nil { logger.Error(ctx, "generate client token failed", "error", err) panic(err) } // init postgresDBClient - postgresDBClient = database.InitPostgresDBInstance(ctx, modelRTConfig.PostgresDBURI) + postgresDBClient = database.InitPostgresDBInstance(modelRTConfig.PostgresDBURI) defer func() { sqlDB, err := postgresDBClient.DB() @@ -127,13 +128,17 @@ func main() { defer parsePool.Release() searchPool, err := util.NewRedigoPool(modelRTConfig.StorageRedisConfig) + if err != nil { + logger.Error(ctx, "init redigo pool failed", "error", err) + panic(err) + } defer searchPool.Close() model.InitAutocompleterWithPool(searchPool) - storageClient := diagram.InitRedisClientInstance(modelRTConfig.StorageRedisConfig, *&modelRTConfig.ServiceConfig.DeployEnv) + storageClient := diagram.InitRedisClientInstance(modelRTConfig.StorageRedisConfig, modelRTConfig.DeployEnv) defer storageClient.Close() - lockerClient := locker.InitClientInstance(modelRTConfig.LockerRedisConfig, *&modelRTConfig.ServiceConfig.DeployEnv) + lockerClient := locker.InitClientInstance(modelRTConfig.LockerRedisConfig, modelRTConfig.DeployEnv) defer lockerClient.Close() // init anchor param ants pool @@ -204,8 +209,10 @@ func main() { return nil }) - // use release mode in productio - // gin.SetMode(gin.ReleaseMode) + // use release mode in production + if modelRTConfig.DeployEnv == constants.ProductionDeployMode { + gin.SetMode(gin.ReleaseMode) + } engine := gin.New() router.RegisterRoutes(engine, serviceToken) @@ -223,7 +230,7 @@ func main() { // } server := http.Server{ - Addr: modelRTConfig.ServiceConfig.ServiceAddr, + Addr: modelRTConfig.ServiceAddr, Handler: engine, } From 02e0c9c31a87d4779955848438620f8654bf053d Mon Sep 17 00:00:00 2001 From: douxu Date: Fri, 30 Jan 2026 17:42:50 +0800 Subject: [PATCH 07/43] optimzie of postgres db code --- database/postgres_init.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/database/postgres_init.go b/database/postgres_init.go index bd7869f..fdba0ee 100644 --- a/database/postgres_init.go +++ b/database/postgres_init.go @@ -13,15 +13,11 @@ import ( var ( postgresOnce sync.Once _globalPostgresClient *gorm.DB - _globalPostgresMu sync.RWMutex ) // GetPostgresDBClient returns the global PostgresDB client.It's safe for concurrent use. func GetPostgresDBClient() *gorm.DB { - _globalPostgresMu.RLock() - client := _globalPostgresClient - _globalPostgresMu.RUnlock() - return client + return _globalPostgresClient } // InitPostgresDBInstance return instance of PostgresDB client From 35cb969a54029fda6d2d1cd82ee8ae8053dffd85 Mon Sep 17 00:00:00 2001 From: douxu Date: Mon, 2 Feb 2026 16:48:46 +0800 Subject: [PATCH 08/43] add code of inter-module communication --- alert/event.go | 58 +++++++++++++ go.mod | 4 +- go.sum | 4 + mq/publisher_with_ssh_091.go | 160 +++++++++++++++++++++++++++++++++++ sharememory/share_memeory.go | 97 --------------------- 5 files changed, 225 insertions(+), 98 deletions(-) create mode 100644 alert/event.go create mode 100644 mq/publisher_with_ssh_091.go delete mode 100644 sharememory/share_memeory.go diff --git a/alert/event.go b/alert/event.go new file mode 100644 index 0000000..f580489 --- /dev/null +++ b/alert/event.go @@ -0,0 +1,58 @@ +// Package alert define alert event struct of modelRT project +package alert + +// EventRecord define struct for CIM event record +type EventRecord struct { + // 事件名称 + Event string `json:"event"` + // 事件唯一标识符 + EventUUID string `json:"event_uuid"` + // 事件类型 + Type int `json:"type"` + // 事件优先级 (0-9) + Priority int `json:"priority"` + // 事件状态 + Status int `json:"status"` + // 可选模板参数 + Category string `json:"category,omitempty"` + // 毫秒级时间戳 (Unix epoch) + Timestamp int64 `json:"timestamp"` + // 事件来源 (station, platform, msa) + From string `json:"from"` + // 事件场景描述对象 (如阈值、当前值) + Condition map[string]any `json:"condition"` + // 与事件相关的订阅信息 + AttachedSubscriptions []any `json:"attached_subscriptions"` + // 事件分析结果对象 + Result map[string]any `json:"result,omitempty"` + // 操作历史记录 (CIM ActivityRecord) + Operations []OperationRecord `json:"operations"` + // 子站告警原始数据 (CIM Alarm 数据) + Alarm map[string]any `json:"alarm,omitempty"` +} + +// OperationRecord 描述对事件的操作记录，如确认(acknowledgment)等 +type OperationRecord struct { + Action string `json:"action"` // 执行的动作，如 "acknowledgment" + Op string `json:"op"` // 操作人/操作账号标识 + TS int64 `json:"ts"` // 操作发生的毫秒时间戳 +} + +// 定义事件类型常量，便于逻辑判断 +const ( + TypeGeneralHard = 0 + TypeGeneralPlatformSoft = 1 + TypeGeneralApplicationSoft = 2 + TypeWarnHard = 3 + TypeWarnPlatformSoft = 4 + TypeWarnApplicationSoft = 5 + TypeCriticalHard = 6 + TypeCriticalPlatformSoft = 7 + TypeCriticalApplicationSoft = 8 +) + +const ( + FromStation = "station" + FromPlatform = "platform" + FromMSA = "msa" +) diff --git a/go.mod b/go.mod index 3695aef..99cc668 100644 --- a/go.mod +++ b/go.mod @@ -13,14 +13,15 @@ require ( github.com/json-iterator/go v1.1.12 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/panjf2000/ants/v2 v2.10.0 + github.com/rabbitmq/amqp091-go v1.10.0 github.com/redis/go-redis/v9 v9.7.3 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.4 + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 go.uber.org/zap v1.27.0 - golang.org/x/sys v0.28.0 gorm.io/driver/mysql v1.5.7 gorm.io/driver/postgres v1.5.9 gorm.io/gorm v1.25.12 @@ -81,6 +82,7 @@ require ( golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.32.0 // indirect golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.28.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/go.sum b/go.sum index 33b982e..2b3c5ff 100644 --- a/go.sum +++ b/go.sum @@ -121,6 +121,8 @@ github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xl github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= @@ -162,6 +164,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= diff --git a/mq/publisher_with_ssh_091.go b/mq/publisher_with_ssh_091.go new file mode 100644 index 0000000..2319558 --- /dev/null +++ b/mq/publisher_with_ssh_091.go @@ -0,0 +1,160 @@ +// Package mq provides read or write access to message queue services +package mq + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "log" + "os" + "time" + + amqp "github.com/rabbitmq/amqp091-go" + "github.com/youmark/pkcs8" +) + +func main() { + exchangeName := "getting-started-go-exchange" + queueName := "getting-started-go-queue" + routingKey := "routing-key" + + deadExchangeName := "getting-started-go-dead-letter-exchange" + deadQueueName := "getting-started-go-dead-letter-queue" + deadRoutingKey := "dead-letter-routing-key" + + caCert, err := os.ReadFile("../certs/ca_certificate.pem") + if err != nil { + log.Fatal("read ca file failed", err) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + keyData, err := os.ReadFile("../certs/client_key.pem") + if err != nil { + log.Fatal("read private key file failed", err) + } + + block, _ := pem.Decode(keyData) + password := []byte("ecl3000") + privateKey, err := pkcs8.ParsePKCS8PrivateKey(block.Bytes, password) + if err != nil { + log.Fatal("parse private key failed", err) + } + + pemBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) + pemBlock := &pem.Block{Type: "PRIVATE KEY", Bytes: pemBytes} + + certPEM, err := os.ReadFile("../certs/client_certificate.pem") + clientCert, err := tls.X509KeyPair(certPEM, pem.EncodeToMemory(pemBlock)) + if err != nil { + log.Fatal("load client cert failed", err) + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{clientCert}, + RootCAs: caCertPool, + InsecureSkipVerify: false, + ServerName: "douxu-buntu22", + } + + url := "amqps://192.168.2.104:5671/" + + conn, err := amqp.DialConfig(url, amqp.Config{ + TLSClientConfig: tlsConfig, + SASL: []amqp.Authentication{&amqp.ExternalAuth{}}, + Heartbeat: 10 * time.Second, + }) + if err != nil { + log.Fatal("Error opening connection: ", err) + } + defer conn.Close() + + ch, err := conn.Channel() + if err != nil { + log.Fatal("Error opening channel: ", err) + } + defer ch.Close() + + go func() { + closeErr := <-conn.NotifyClose(make(chan *amqp.Error)) + log.Printf("Connection closed: %v", closeErr) + }() + + err = ch.ExchangeDeclare(deadExchangeName, "topic", true, false, false, false, nil) + if err != nil { + log.Fatal("Error declaring dead letter exchange: ", err) + } + + _, err = ch.QueueDeclare(deadQueueName, true, false, false, false, nil) + if err != nil { + log.Fatal("Error declaring dead letter queue: ", err) + } + + err = ch.QueueBind(deadQueueName, deadRoutingKey, deadExchangeName, false, nil) + if err != nil { + log.Fatal("Error binding dead letter: ", err) + } + + err = ch.ExchangeDeclare(exchangeName, "topic", true, false, false, false, nil) + if err != nil { + log.Fatal("Error declaring exchange: ", err) + } + + args := amqp.Table{ + "x-max-length": int32(50), + "x-dead-letter-exchange": deadExchangeName, + "x-dead-letter-routing-key": deadRoutingKey, + } + _, err = ch.QueueDeclare(queueName, true, false, false, false, args) + if err != nil { + log.Fatal("Error declaring queue: ", err) + } + + err = ch.QueueBind(queueName, routingKey, exchangeName, false, nil) + if err != nil { + log.Fatal("Error binding queue: ", err) + } + + if err := ch.Confirm(false); err != nil { + log.Fatal("Channel could not be put into confirm mode: ", err) + } + + confirms := ch.NotifyPublish(make(chan amqp.Confirmation, 1)) + + for i := 0; i < 100; i++ { + msgBody := fmt.Sprintf("Hello, World!%d", i) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err = ch.PublishWithContext(ctx, + exchangeName, // exchange + routingKey, // routing key + false, // mandatory + false, // immediate + amqp.Publishing{ + ContentType: "text/plain", + Body: []byte(msgBody), + }) + cancel() + + if err != nil { + log.Printf("Error publishing message: %v", err) + time.Sleep(1 * time.Second) + continue + } + + select { + case confirm := <-confirms: + if confirm.Ack { + log.Printf("[Publisher] Message %d accepted", i) + } else { + log.Printf("[Publisher] Message %d Nack (Rejected by RabbitMQ)", i) + } + case <-time.After(5 * time.Second): + log.Printf("[Publisher] Timeout waiting for confirm for message %d", i) + } + } + + log.Println("Producer finished sending messages") +} diff --git a/sharememory/share_memeory.go b/sharememory/share_memeory.go deleted file mode 100644 index 8248064..0000000 --- a/sharememory/share_memeory.go +++ /dev/null @@ -1,97 +0,0 @@ -package sharememory - -import ( - "fmt" - "unsafe" - - "modelRT/orm" - - "golang.org/x/sys/unix" -) - -// CreateShareMemory defines a function to create a shared memory -func CreateShareMemory(key uintptr, structSize uintptr) (uintptr, error) { - // logger := logger.GetLoggerInstance() - // create shared memory - shmID, _, err := unix.Syscall(unix.SYS_SHMGET, key, structSize, unix.IPC_CREAT|0o666) - if err != 0 { - // logger.Error(fmt.Sprintf("create shared memory by key %v failed:", key), zap.Error(err)) - return 0, fmt.Errorf("create shared memory failed:%w", err) - } - - // attach shared memory - shmAddr, _, err := unix.Syscall(unix.SYS_SHMAT, shmID, 0, 0) - if err != 0 { - // logger.Error(fmt.Sprintf("attach shared memory by shmID %v failed:", shmID), zap.Error(err)) - return 0, fmt.Errorf("attach shared memory failed:%w", err) - } - return shmAddr, nil -} - -// ReadComponentFromShareMemory defines a function to read component value from shared memory -func ReadComponentFromShareMemory(key uintptr, componentInfo *orm.Component) error { - structSize := unsafe.Sizeof(orm.Component{}) - shmID, _, err := unix.Syscall(unix.SYS_SHMGET, key, uintptr(int(structSize)), 0o666) - if err != 0 { - return fmt.Errorf("get shared memory failed:%w", err) - } - - shmAddr, _, err := unix.Syscall(unix.SYS_SHMAT, shmID, 0, 0) - if err != 0 { - return fmt.Errorf("attach shared memory failed:%w", err) - } - - // 读取共享内存中的数据 - componentInfo = (*orm.Component)(unsafe.Pointer(shmAddr + structSize)) - - // Detach shared memory - unix.Syscall(unix.SYS_SHMDT, shmAddr, 0, 0) - return nil -} - -func WriteComponentInShareMemory(key uintptr, componentInfo *orm.Component) error { - structSize := unsafe.Sizeof(orm.Component{}) - shmID, _, err := unix.Syscall(unix.SYS_SHMGET, key, uintptr(int(structSize)), 0o666) - if err != 0 { - return fmt.Errorf("get shared memory failed:%w", err) - } - - shmAddr, _, err := unix.Syscall(unix.SYS_SHMAT, shmID, 0, 0) - if err != 0 { - return fmt.Errorf("attach shared memory failed:%w", err) - } - - obj := (*orm.Component)(unsafe.Pointer(shmAddr + unsafe.Sizeof(structSize))) - fmt.Println(obj) - - // id integer NOT NULL DEFAULT nextval('component_id_seq'::regclass), - // global_uuid uuid NOT NULL DEFAULT gen_random_uuid(), - // nspath character varying(32) COLLATE pg_catalog."default", - // tag character varying(32) COLLATE pg_catalog."default" NOT NULL, - // name character varying(64) COLLATE pg_catalog."default" NOT NULL, - // description character varying(512) COLLATE pg_catalog."default" NOT NULL DEFAULT ''::character varying, - // grid character varying(64) COLLATE pg_catalog."default" NOT NULL, - // zone character varying(64) COLLATE pg_catalog."default" NOT NULL, - // station character varying(64) COLLATE pg_catalog."default" NOT NULL, - // type integer NOT NULL, - // in_service boolean DEFAULT false, - // state integer NOT NULL DEFAULT 0, - // connected_bus jsonb NOT NULL DEFAULT '{}'::jsonb, - // label jsonb NOT NULL DEFAULT '{}'::jsonb, - // context jsonb NOT NULL DEFAULT '{}'::jsonb, - // page_id integer NOT NULL, - // op integer NOT NULL DEFAULT '-1'::integer, - // ts timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, - - unix.Syscall(unix.SYS_SHMDT, shmAddr, 0, 0) - return nil -} - -// DeleteShareMemory defines a function to delete shared memory -func DeleteShareMemory(key uintptr) error { - _, _, err := unix.Syscall(unix.SYS_SHM_UNLINK, key, 0, 0o666) - if err != 0 { - return fmt.Errorf("get shared memory failed:%w", err) - } - return nil -} From 9be984899cace942cccfeb72ec96213238bb034c Mon Sep 17 00:00:00 2001 From: douxu Date: Tue, 3 Feb 2026 17:05:32 +0800 Subject: [PATCH 09/43] optimize code of push event alarm func --- config/config.go | 15 ++ handler/real_time_data_pull.go | 1 - handler/real_time_data_query.go | 1 - handler/real_time_data_subscription.go | 1 - main.go | 4 + mq/publish_event.go | 129 +++++++++++++++++ mq/publisher_with_ssh_091.go | 160 --------------------- mq/rabbitmq_init.go | 185 +++++++++++++++++++++++++ 8 files changed, 333 insertions(+), 163 deletions(-) create mode 100644 mq/publish_event.go delete mode 100644 mq/publisher_with_ssh_091.go create mode 100644 mq/rabbitmq_init.go diff --git a/config/config.go b/config/config.go index b6379d3..3c09187 100644 --- a/config/config.go +++ b/config/config.go @@ -22,6 +22,20 @@ type ServiceConfig struct { DeployEnv string `mapstructure:"deploy_env"` } +// RabbitMQConfig define config struct of RabbitMQ config +type RabbitMQConfig struct { + CACertPath string `mapstructure:"ca_cert_path"` + ClientKeyPath string `mapstructure:"client_key_path"` + ClientKeyPassword string `mapstructure:"client_key_password"` + ClientCertPath string `mapstructure:"client_cert_path"` + InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"` + ServerName string `mapstructure:"server_name"` + User string `mapstructure:"user"` + Password string `mapstructure:"password"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` +} + // KafkaConfig define config struct of kafka config type KafkaConfig struct { Servers string `mapstructure:"Servers"` @@ -82,6 +96,7 @@ type ModelRTConfig struct { BaseConfig `mapstructure:"base"` ServiceConfig `mapstructure:"service"` PostgresConfig `mapstructure:"postgres"` + RabbitMQConfig `mapstructure:"rabbitmq"` KafkaConfig `mapstructure:"kafka"` LoggerConfig `mapstructure:"logger"` AntsConfig `mapstructure:"ants"` diff --git a/handler/real_time_data_pull.go b/handler/real_time_data_pull.go index 24ccf50..986ed11 100644 --- a/handler/real_time_data_pull.go +++ b/handler/real_time_data_pull.go @@ -401,7 +401,6 @@ func stopAllPolling(ctx context.Context, stopChanMap map[string]chan struct{}) { close(stopChan) } clear(stopChanMap) - return } // redisPollingConfig define struct for param which query real time data from redis diff --git a/handler/real_time_data_query.go b/handler/real_time_data_query.go index c734739..03e9786 100644 --- a/handler/real_time_data_query.go +++ b/handler/real_time_data_query.go @@ -168,7 +168,6 @@ func receiveRealTimeDataByWebSocket(ctx context.Context, params url.Values, tran } transportChannel <- subPoss } - return } // messageTypeToString define func of auxiliary to convert message type to string diff --git a/handler/real_time_data_subscription.go b/handler/real_time_data_subscription.go index afc396b..3f171cf 100644 --- a/handler/real_time_data_subscription.go +++ b/handler/real_time_data_subscription.go @@ -620,7 +620,6 @@ func (s *SharedSubState) RemoveTargets(ctx context.Context, clientID string, mea // UpdateTargets define function to update targets in SharedSubState func (s *SharedSubState) UpdateTargets(ctx context.Context, tx *gorm.DB, clientID string, measurements []network.RealTimeMeasurementItem) ([]network.TargetResult, error) { requestTargetsCount := processRealTimeRequestCount(measurements) - targetProcessResults := make([]network.TargetResult, 0, requestTargetsCount) s.globalMutex.RLock() config, exist := s.subMap[clientID] diff --git a/main.go b/main.go index a0aca1e..8658a56 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ import ( "modelRT/diagram" "modelRT/logger" "modelRT/model" + "modelRT/mq" "modelRT/pool" "modelRT/router" "modelRT/util" @@ -149,6 +150,9 @@ func main() { } defer anchorRealTimePool.Release() + // init rabbitmq connection + mq.InitRabbitProxy(ctx, modelRTConfig.RabbitMQConfig) + postgresDBClient.Transaction(func(tx *gorm.DB) error { // load circuit diagram from postgres // componentTypeMap, err := database.QueryCircuitDiagramComponentFromDB(cancelCtx, tx, parsePool) diff --git a/mq/publish_event.go b/mq/publish_event.go new file mode 100644 index 0000000..7984741 --- /dev/null +++ b/mq/publish_event.go @@ -0,0 +1,129 @@ +// Package mq provides read or write access to message queue services +package mq + +import ( + "context" + "time" + + "modelRT/logger" + + amqp "github.com/rabbitmq/amqp091-go" +) + +const ( + routingKey = "event-alarm-routing-key" + exchangeName = "event-alarm-exchange" + queueName = "event-alarm-queue" + deadRoutingKey = "event-alarm-dead-letter-routing-key" + deadExchangeName = "event-alarm-dead-letter-exchange" + deadQueueName = "event-alarm-dead-letter-queue" +) + +func initChannel(ctx context.Context) (*amqp.Channel, error) { + var channel *amqp.Channel + var err error + + channel, err = GetConn().Channel() + if err != nil { + logger.Error(ctx, "open rabbitMQ server channel failed", "error", err) + } + + err = channel.ExchangeDeclare(deadExchangeName, "topic", true, false, false, false, nil) + if err != nil { + logger.Error(ctx, "declare event dead letter exchange failed", "error", err) + } + + _, err = channel.QueueDeclare(deadQueueName, true, false, false, false, nil) + if err != nil { + logger.Error(ctx, "declare event dead letter queue failed", "error", err) + } + + err = channel.QueueBind(deadQueueName, deadRoutingKey, deadExchangeName, false, nil) + if err != nil { + logger.Error(ctx, "bind event dead letter queue with routing key and exchange failed", "error", err) + } + + err = channel.ExchangeDeclare(exchangeName, "topic", true, false, false, false, nil) + if err != nil { + logger.Error(ctx, "declare event exchange failed", "error", err) + } + + args := amqp.Table{ + // messages that accumulate to the maximum number will be automatically transferred to the dead letter queue + "x-max-length": int32(50), + "x-dead-letter-exchange": deadExchangeName, + "x-dead-letter-routing-key": deadRoutingKey, + } + _, err = channel.QueueDeclare(queueName, true, false, false, false, args) + if err != nil { + logger.Error(ctx, "declare event queue failed", "error", err) + } + + err = channel.QueueBind(queueName, routingKey, exchangeName, false, nil) + if err != nil { + logger.Error(ctx, "bind event queue with routing key and exchange failed:", "error", err) + } + + if err := channel.Confirm(false); err != nil { + logger.Error(ctx, "channel could not be put into confirm mode", "error", err) + } + return channel, nil +} + +func pushEventToRabbitMQ(ctx context.Context, msgChan chan string) { + channel, err := initChannel(ctx) + if err != nil { + logger.Error(ctx, "initializing rabbitMQ channel failed", "error", err) + return + } + // TODO 使用配置修改确认模式通道参数 + confirms := channel.NotifyPublish(make(chan amqp.Confirmation, 100)) + + go func() { + for { + select { + case confirm, ok := <-confirms: + if !ok { + return + } + if !confirm.Ack { + logger.Error(ctx, "publish message failed (rejected by rabbitMQ)", "tag", confirm.DeliveryTag) + } + case <-ctx.Done(): + return + } + } + }() + + for { + select { + case <-ctx.Done(): + logger.Info(ctx, "push event alarm message to rabbitMQ stopped by context cancel") + channel.Close() + return + case msg, ok := <-msgChan: + if !ok { + logger.Info(ctx, "push event alarm message to rabbitMQ stopped by msgChan closed, exiting push loop") + channel.Close() + return + } + + // send event alarm message to rabbitMQ queue + pubCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + err = channel.PublishWithContext(pubCtx, + exchangeName, // exchange + routingKey, // routing key + false, // mandatory + false, // immediate + amqp.Publishing{ + ContentType: "text/plain", + Body: []byte(msg), + }) + cancel() + + if err != nil { + logger.Error(ctx, "publish message to rabbitMQ queue failed", "message", msg, "error", err) + } + } + } +} diff --git a/mq/publisher_with_ssh_091.go b/mq/publisher_with_ssh_091.go deleted file mode 100644 index 2319558..0000000 --- a/mq/publisher_with_ssh_091.go +++ /dev/null @@ -1,160 +0,0 @@ -// Package mq provides read or write access to message queue services -package mq - -import ( - "context" - "crypto/tls" - "crypto/x509" - "encoding/pem" - "fmt" - "log" - "os" - "time" - - amqp "github.com/rabbitmq/amqp091-go" - "github.com/youmark/pkcs8" -) - -func main() { - exchangeName := "getting-started-go-exchange" - queueName := "getting-started-go-queue" - routingKey := "routing-key" - - deadExchangeName := "getting-started-go-dead-letter-exchange" - deadQueueName := "getting-started-go-dead-letter-queue" - deadRoutingKey := "dead-letter-routing-key" - - caCert, err := os.ReadFile("../certs/ca_certificate.pem") - if err != nil { - log.Fatal("read ca file failed", err) - } - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - - keyData, err := os.ReadFile("../certs/client_key.pem") - if err != nil { - log.Fatal("read private key file failed", err) - } - - block, _ := pem.Decode(keyData) - password := []byte("ecl3000") - privateKey, err := pkcs8.ParsePKCS8PrivateKey(block.Bytes, password) - if err != nil { - log.Fatal("parse private key failed", err) - } - - pemBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) - pemBlock := &pem.Block{Type: "PRIVATE KEY", Bytes: pemBytes} - - certPEM, err := os.ReadFile("../certs/client_certificate.pem") - clientCert, err := tls.X509KeyPair(certPEM, pem.EncodeToMemory(pemBlock)) - if err != nil { - log.Fatal("load client cert failed", err) - } - - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{clientCert}, - RootCAs: caCertPool, - InsecureSkipVerify: false, - ServerName: "douxu-buntu22", - } - - url := "amqps://192.168.2.104:5671/" - - conn, err := amqp.DialConfig(url, amqp.Config{ - TLSClientConfig: tlsConfig, - SASL: []amqp.Authentication{&amqp.ExternalAuth{}}, - Heartbeat: 10 * time.Second, - }) - if err != nil { - log.Fatal("Error opening connection: ", err) - } - defer conn.Close() - - ch, err := conn.Channel() - if err != nil { - log.Fatal("Error opening channel: ", err) - } - defer ch.Close() - - go func() { - closeErr := <-conn.NotifyClose(make(chan *amqp.Error)) - log.Printf("Connection closed: %v", closeErr) - }() - - err = ch.ExchangeDeclare(deadExchangeName, "topic", true, false, false, false, nil) - if err != nil { - log.Fatal("Error declaring dead letter exchange: ", err) - } - - _, err = ch.QueueDeclare(deadQueueName, true, false, false, false, nil) - if err != nil { - log.Fatal("Error declaring dead letter queue: ", err) - } - - err = ch.QueueBind(deadQueueName, deadRoutingKey, deadExchangeName, false, nil) - if err != nil { - log.Fatal("Error binding dead letter: ", err) - } - - err = ch.ExchangeDeclare(exchangeName, "topic", true, false, false, false, nil) - if err != nil { - log.Fatal("Error declaring exchange: ", err) - } - - args := amqp.Table{ - "x-max-length": int32(50), - "x-dead-letter-exchange": deadExchangeName, - "x-dead-letter-routing-key": deadRoutingKey, - } - _, err = ch.QueueDeclare(queueName, true, false, false, false, args) - if err != nil { - log.Fatal("Error declaring queue: ", err) - } - - err = ch.QueueBind(queueName, routingKey, exchangeName, false, nil) - if err != nil { - log.Fatal("Error binding queue: ", err) - } - - if err := ch.Confirm(false); err != nil { - log.Fatal("Channel could not be put into confirm mode: ", err) - } - - confirms := ch.NotifyPublish(make(chan amqp.Confirmation, 1)) - - for i := 0; i < 100; i++ { - msgBody := fmt.Sprintf("Hello, World!%d", i) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - err = ch.PublishWithContext(ctx, - exchangeName, // exchange - routingKey, // routing key - false, // mandatory - false, // immediate - amqp.Publishing{ - ContentType: "text/plain", - Body: []byte(msgBody), - }) - cancel() - - if err != nil { - log.Printf("Error publishing message: %v", err) - time.Sleep(1 * time.Second) - continue - } - - select { - case confirm := <-confirms: - if confirm.Ack { - log.Printf("[Publisher] Message %d accepted", i) - } else { - log.Printf("[Publisher] Message %d Nack (Rejected by RabbitMQ)", i) - } - case <-time.After(5 * time.Second): - log.Printf("[Publisher] Timeout waiting for confirm for message %d", i) - } - } - - log.Println("Producer finished sending messages") -} diff --git a/mq/rabbitmq_init.go b/mq/rabbitmq_init.go new file mode 100644 index 0000000..9866f93 --- /dev/null +++ b/mq/rabbitmq_init.go @@ -0,0 +1,185 @@ +// Package mq define message queue operation functions +package mq + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "net/url" + "os" + "sync" + "time" + + "modelRT/config" + "modelRT/logger" + + amqp "github.com/rabbitmq/amqp091-go" + "github.com/youmark/pkcs8" +) + +var ( + _globalRabbitMQProxy *RabbitMQProxy + rabbitMQOnce sync.Once +) + +// RabbitMQProxy define stuct of rabbitMQ connection proxy +type RabbitMQProxy struct { + Conn *amqp.Connection + mu sync.Mutex +} + +// RabbitMQCertConf define stuct of rabbitMQ connection certificates config +type RabbitMQCertConf struct { + serverName string + insecureSkipVerify bool + clientCert tls.Certificate + caCertPool *x509.CertPool +} + +// GetConn define func to return the rabbitMQ connection +func GetConn() *amqp.Connection { + _globalRabbitMQProxy.mu.Lock() + defer _globalRabbitMQProxy.mu.Unlock() + return _globalRabbitMQProxy.Conn +} + +// InitRabbitProxy return instance of rabbitMQ connection +func InitRabbitProxy(ctx context.Context, rCfg config.RabbitMQConfig) *RabbitMQProxy { + amqpURI := generateRabbitMQURI(rCfg) + certConf, err := readCertFiles(ctx, rCfg) + if err != nil { + logger.Error(ctx, "read rabbitMQ cert files failed", "error", err) + panic(err) + } + rabbitMQOnce.Do(func() { + conn := initRabbitMQ(ctx, amqpURI, certConf) + _globalRabbitMQProxy = &RabbitMQProxy{Conn: conn} + go _globalRabbitMQProxy.handleReconnect(ctx, amqpURI) + }) + return _globalRabbitMQProxy +} + +// initRabbitMQ return instance of rabbitMQ connection +func initRabbitMQ(ctx context.Context, rabbitMQURI string, certConf *RabbitMQCertConf) *amqp.Connection { + logger.Info(ctx, fmt.Sprintf("connecting to rabbitMQ server at: %s", rabbitMQURI)) + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{certConf.clientCert}, + RootCAs: certConf.caCertPool, + InsecureSkipVerify: certConf.insecureSkipVerify, + ServerName: certConf.serverName, + } + + conn, err := amqp.DialConfig(rabbitMQURI, amqp.Config{ + TLSClientConfig: tlsConfig, + SASL: []amqp.Authentication{&amqp.ExternalAuth{}}, + Heartbeat: 10 * time.Second, + }) + if err != nil { + logger.Error(ctx, "Error opening connection: ", "error", err) + } + defer conn.Close() + + return conn +} + +func (p *RabbitMQProxy) handleReconnect(ctx context.Context, rabbitMQURI string) { + for { + closeChan := make(chan *amqp.Error) + GetConn().NotifyClose(closeChan) + + err, ok := <-closeChan + + if !ok { + logger.Info(ctx, "rabbitMQ notify channel closed, stopping solicitor") + break + } + + if err != nil { + logger.Warn(ctx, "rabbitMQ connection closed by error", "reason", err) + } else { + logger.Info(ctx, "rabbitMQ connection closed normally (nil err)") + } + + for { + time.Sleep(5 * time.Second) + newConn, err := amqp.Dial(rabbitMQURI) + if err == nil { + p.mu.Lock() + p.Conn = newConn + p.mu.Unlock() + logger.Info(ctx, "rabbitMQ reconnected successfully") + break + } + logger.Error(ctx, "rabbitMQ reconnect failed", "err", err) + } + } +} + +func generateRabbitMQURI(rCfg config.RabbitMQConfig) string { + user := url.QueryEscape(rCfg.User) + password := url.QueryEscape(rCfg.Password) + + amqpURI := fmt.Sprintf("amqp://%s:%s@%s:%d/", + user, + password, + rCfg.Host, + rCfg.Port, + ) + return amqpURI +} + +func readCertFiles(ctx context.Context, rCfg config.RabbitMQConfig) (*RabbitMQCertConf, error) { + var initFailedFlag bool + certConf := RabbitMQCertConf{ + insecureSkipVerify: rCfg.InsecureSkipVerify, + } + + caCert, err := os.ReadFile(rCfg.CACertPath) + if err != nil { + logger.Error(ctx, "read server ca file failed", "error", err) + initFailedFlag = true + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + certConf.caCertPool = caCertPool + + keyData, err := os.ReadFile(rCfg.ClientKeyPath) + if err != nil { + logger.Error(ctx, "read private key file failed", "error", err) + initFailedFlag = true + } + + block, _ := pem.Decode(keyData) + privateKey, err := pkcs8.ParsePKCS8PrivateKey(block.Bytes, []byte(rCfg.ClientKeyPassword)) + if err != nil { + logger.Error(ctx, "parse private key failed", "error", err) + initFailedFlag = true + } + + pemBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + logger.Error(ctx, "parse private key failed", "error", err) + initFailedFlag = true + } + pemBlock := &pem.Block{Type: "PRIVATE KEY", Bytes: pemBytes} + + certPEM, err := os.ReadFile(rCfg.ClientCertPath) + if err != nil { + logger.Error(ctx, "parse private key failed", "error", err) + initFailedFlag = true + } + clientCert, err := tls.X509KeyPair(certPEM, pem.EncodeToMemory(pemBlock)) + if err != nil { + logger.Error(ctx, "load client cert failed", "error", err) + initFailedFlag = true + } + certConf.serverName = rCfg.ServerName + certConf.clientCert = clientCert + if initFailedFlag { + return nil, fmt.Errorf("rabbitMQ cert files init failed") + } + return &certConf, nil +} From f45b7d5fa4cb7130f9b8466e8a047d44cca723f9 Mon Sep 17 00:00:00 2001 From: douxu Date: Wed, 4 Feb 2026 17:43:09 +0800 Subject: [PATCH 10/43] optimize code of init rabbitmq connect func --- mq/rabbitmq_init.go | 79 ++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/mq/rabbitmq_init.go b/mq/rabbitmq_init.go index 9866f93..9eaa9f5 100644 --- a/mq/rabbitmq_init.go +++ b/mq/rabbitmq_init.go @@ -48,9 +48,9 @@ func GetConn() *amqp.Connection { // InitRabbitProxy return instance of rabbitMQ connection func InitRabbitProxy(ctx context.Context, rCfg config.RabbitMQConfig) *RabbitMQProxy { amqpURI := generateRabbitMQURI(rCfg) - certConf, err := readCertFiles(ctx, rCfg) + certConf, err := initCertConf(rCfg) if err != nil { - logger.Error(ctx, "read rabbitMQ cert files failed", "error", err) + logger.Error(ctx, "init rabbitMQ cert config failed", "error", err) panic(err) } rabbitMQOnce.Do(func() { @@ -63,7 +63,7 @@ func InitRabbitProxy(ctx context.Context, rCfg config.RabbitMQConfig) *RabbitMQP // initRabbitMQ return instance of rabbitMQ connection func initRabbitMQ(ctx context.Context, rabbitMQURI string, certConf *RabbitMQCertConf) *amqp.Connection { - logger.Info(ctx, fmt.Sprintf("connecting to rabbitMQ server at: %s", rabbitMQURI)) + logger.Info(ctx, "connecting to rabbitMQ server", "rabbit_uri", rabbitMQURI) tlsConfig := &tls.Config{ Certificates: []tls.Certificate{certConf.clientCert}, @@ -78,7 +78,7 @@ func initRabbitMQ(ctx context.Context, rabbitMQURI string, certConf *RabbitMQCer Heartbeat: 10 * time.Second, }) if err != nil { - logger.Error(ctx, "Error opening connection: ", "error", err) + logger.Error(ctx, "init rabbitMQ connection failed", "error", err) } defer conn.Close() @@ -131,55 +131,54 @@ func generateRabbitMQURI(rCfg config.RabbitMQConfig) string { return amqpURI } -func readCertFiles(ctx context.Context, rCfg config.RabbitMQConfig) (*RabbitMQCertConf, error) { - var initFailedFlag bool - certConf := RabbitMQCertConf{ +func initCertConf(rCfg config.RabbitMQConfig) (*RabbitMQCertConf, error) { + certConf := &RabbitMQCertConf{ insecureSkipVerify: rCfg.InsecureSkipVerify, + serverName: rCfg.ServerName, } caCert, err := os.ReadFile(rCfg.CACertPath) if err != nil { - logger.Error(ctx, "read server ca file failed", "error", err) - initFailedFlag = true + return nil, fmt.Errorf("read server ca file failed: %w", err) } caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) + if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { + return nil, fmt.Errorf("failed to parse root certificate from %s", rCfg.CACertPath) + } certConf.caCertPool = caCertPool - keyData, err := os.ReadFile(rCfg.ClientKeyPath) - if err != nil { - logger.Error(ctx, "read private key file failed", "error", err) - initFailedFlag = true - } - - block, _ := pem.Decode(keyData) - privateKey, err := pkcs8.ParsePKCS8PrivateKey(block.Bytes, []byte(rCfg.ClientKeyPassword)) - if err != nil { - logger.Error(ctx, "parse private key failed", "error", err) - initFailedFlag = true - } - - pemBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) - if err != nil { - logger.Error(ctx, "parse private key failed", "error", err) - initFailedFlag = true - } - pemBlock := &pem.Block{Type: "PRIVATE KEY", Bytes: pemBytes} - certPEM, err := os.ReadFile(rCfg.ClientCertPath) if err != nil { - logger.Error(ctx, "parse private key failed", "error", err) - initFailedFlag = true + return nil, fmt.Errorf("read client cert file failed: %w", err) } - clientCert, err := tls.X509KeyPair(certPEM, pem.EncodeToMemory(pemBlock)) + + keyData, err := os.ReadFile(rCfg.ClientKeyPath) if err != nil { - logger.Error(ctx, "load client cert failed", "error", err) - initFailedFlag = true + return nil, fmt.Errorf("read private key file failed: %w", err) } - certConf.serverName = rCfg.ServerName + + block, _ := pem.Decode(keyData) + if block == nil { + return nil, fmt.Errorf("failed to decode PEM block from private key") + } + + der, err := pkcs8.ParsePKCS8PrivateKey(block.Bytes, []byte(rCfg.ClientKeyPassword)) + if err != nil { + return nil, fmt.Errorf("parse password-protected private key failed: %w", err) + } + + privBytes, err := x509.MarshalPKCS8PrivateKey(der) + if err != nil { + return nil, fmt.Errorf("marshal private key failed: %w", err) + } + + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) + + clientCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, fmt.Errorf("create x509 key pair failed: %w", err) + } + certConf.clientCert = clientCert - if initFailedFlag { - return nil, fmt.Errorf("rabbitMQ cert files init failed") - } - return &certConf, nil + return certConf, nil } From 581153ed8d5b964e82c52b873b9d56cfebbd04e9 Mon Sep 17 00:00:00 2001 From: douxu Date: Thu, 5 Feb 2026 17:01:16 +0800 Subject: [PATCH 11/43] add git ignore item of mask certificate files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 51b9af9..673582e 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ go.work /log/ # Shield config files in the configs folder /configs/**/*.yaml +/configs/**/*.pem From 6618209bcc3709abd5164a2ee89d3087735a0b89 Mon Sep 17 00:00:00 2001 From: douxu Date: Fri, 6 Feb 2026 17:45:59 +0800 Subject: [PATCH 12/43] optimzie code of rabbitmq connection --- main.go | 3 + mq/rabbitmq_init.go | 133 +++++++++++++-------- real-time-data/real_time_data_computing.go | 2 +- 3 files changed, 85 insertions(+), 53 deletions(-) diff --git a/main.go b/main.go index 8658a56..8cc8a76 100644 --- a/main.go +++ b/main.go @@ -243,9 +243,12 @@ func main() { signal.Notify(done, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) go func() { <-done + logger.Info(ctx, "shutdown signal received, cleaning up...") if err := server.Shutdown(context.Background()); err != nil { logger.Error(ctx, "shutdown serverError", "err", err) } + mq.CloseRabbitProxy() + logger.Info(ctx, "resources cleaned up, exiting") }() logger.Info(ctx, "starting ModelRT server") diff --git a/mq/rabbitmq_init.go b/mq/rabbitmq_init.go index 9eaa9f5..511547a 100644 --- a/mq/rabbitmq_init.go +++ b/mq/rabbitmq_init.go @@ -26,12 +26,14 @@ var ( // RabbitMQProxy define stuct of rabbitMQ connection proxy type RabbitMQProxy struct { - Conn *amqp.Connection - mu sync.Mutex + tlsConf *tls.Config + conn *amqp.Connection + cancel context.CancelFunc // 增加这个用于停止重连协程 + mu sync.Mutex } -// RabbitMQCertConf define stuct of rabbitMQ connection certificates config -type RabbitMQCertConf struct { +// rabbitMQCertConf define stuct of rabbitMQ connection certificates config +type rabbitMQCertConf struct { serverName string insecureSkipVerify bool clientCert tls.Certificate @@ -42,45 +44,38 @@ type RabbitMQCertConf struct { func GetConn() *amqp.Connection { _globalRabbitMQProxy.mu.Lock() defer _globalRabbitMQProxy.mu.Unlock() - return _globalRabbitMQProxy.Conn + return _globalRabbitMQProxy.conn } // InitRabbitProxy return instance of rabbitMQ connection func InitRabbitProxy(ctx context.Context, rCfg config.RabbitMQConfig) *RabbitMQProxy { amqpURI := generateRabbitMQURI(rCfg) - certConf, err := initCertConf(rCfg) + tlsConf, err := initCertConf(rCfg) if err != nil { logger.Error(ctx, "init rabbitMQ cert config failed", "error", err) panic(err) } rabbitMQOnce.Do(func() { - conn := initRabbitMQ(ctx, amqpURI, certConf) - _globalRabbitMQProxy = &RabbitMQProxy{Conn: conn} - go _globalRabbitMQProxy.handleReconnect(ctx, amqpURI) + cancelCtx, cancel := context.WithCancel(ctx) + conn := initRabbitMQ(ctx, amqpURI, tlsConf) + _globalRabbitMQProxy = &RabbitMQProxy{tlsConf: tlsConf, conn: conn, cancel: cancel} + go _globalRabbitMQProxy.handleReconnect(cancelCtx, amqpURI) }) return _globalRabbitMQProxy } // initRabbitMQ return instance of rabbitMQ connection -func initRabbitMQ(ctx context.Context, rabbitMQURI string, certConf *RabbitMQCertConf) *amqp.Connection { - logger.Info(ctx, "connecting to rabbitMQ server", "rabbit_uri", rabbitMQURI) - - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{certConf.clientCert}, - RootCAs: certConf.caCertPool, - InsecureSkipVerify: certConf.insecureSkipVerify, - ServerName: certConf.serverName, - } - +func initRabbitMQ(ctx context.Context, rabbitMQURI string, tlsConf *tls.Config) *amqp.Connection { + logger.Info(ctx, "connecting to rabbitMQ server", "rabbitmq_uri", rabbitMQURI) conn, err := amqp.DialConfig(rabbitMQURI, amqp.Config{ - TLSClientConfig: tlsConfig, + TLSClientConfig: tlsConf, SASL: []amqp.Authentication{&amqp.ExternalAuth{}}, Heartbeat: 10 * time.Second, }) if err != nil { logger.Error(ctx, "init rabbitMQ connection failed", "error", err) + panic(err) } - defer conn.Close() return conn } @@ -90,31 +85,65 @@ func (p *RabbitMQProxy) handleReconnect(ctx context.Context, rabbitMQURI string) closeChan := make(chan *amqp.Error) GetConn().NotifyClose(closeChan) - err, ok := <-closeChan - - if !ok { - logger.Info(ctx, "rabbitMQ notify channel closed, stopping solicitor") - break - } - - if err != nil { - logger.Warn(ctx, "rabbitMQ connection closed by error", "reason", err) - } else { - logger.Info(ctx, "rabbitMQ connection closed normally (nil err)") - } - - for { - time.Sleep(5 * time.Second) - newConn, err := amqp.Dial(rabbitMQURI) - if err == nil { - p.mu.Lock() - p.Conn = newConn - p.mu.Unlock() - logger.Info(ctx, "rabbitMQ reconnected successfully") - break + select { + case <-ctx.Done(): + logger.Info(ctx, "context cancelled, exiting handleReconnect") + return + case err, ok := <-closeChan: + if !ok { + logger.Info(ctx, "rabbitMQ notify channel closed") + return } - logger.Error(ctx, "rabbitMQ reconnect failed", "err", err) + + if err == nil { + logger.Info(ctx, "rabbitMQ connection closed normally, no need to reconnect") + return + } + + logger.Warn(ctx, "rabbitMQ connection closed by error, starting reconnect", "reason", err) } + + if !p.reconnect(ctx, rabbitMQURI) { + return + } + } +} + +func (p *RabbitMQProxy) reconnect(ctx context.Context, rabbitMQURI string) bool { + for { + logger.Info(ctx, "attempting to reconnect to rabbitMQ...") + select { + case <-ctx.Done(): + return false + case <-time.After(5 * time.Second): + + } + + newConn, err := amqp.DialConfig(rabbitMQURI, amqp.Config{ + TLSClientConfig: p.tlsConf, + SASL: []amqp.Authentication{&amqp.ExternalAuth{}}, + Heartbeat: 10 * time.Second, + }) + if err == nil { + p.mu.Lock() + p.conn = newConn + p.mu.Unlock() + logger.Info(ctx, "rabbitMQ reconnected successfully") + return true + } + logger.Error(ctx, "rabbitMQ reconnect failed, will retry", "err", err) + } +} + +// CloseRabbitProxy close the rabbitMQ connection and stop reconnect goroutine +func CloseRabbitProxy() { + if _globalRabbitMQProxy != nil { + _globalRabbitMQProxy.cancel() + _globalRabbitMQProxy.mu.Lock() + if _globalRabbitMQProxy.conn != nil { + _globalRabbitMQProxy.conn.Close() + } + _globalRabbitMQProxy.mu.Unlock() } } @@ -122,7 +151,7 @@ func generateRabbitMQURI(rCfg config.RabbitMQConfig) string { user := url.QueryEscape(rCfg.User) password := url.QueryEscape(rCfg.Password) - amqpURI := fmt.Sprintf("amqp://%s:%s@%s:%d/", + amqpURI := fmt.Sprintf("amqps://%s:%s@%s:%d/", user, password, rCfg.Host, @@ -131,10 +160,10 @@ func generateRabbitMQURI(rCfg config.RabbitMQConfig) string { return amqpURI } -func initCertConf(rCfg config.RabbitMQConfig) (*RabbitMQCertConf, error) { - certConf := &RabbitMQCertConf{ - insecureSkipVerify: rCfg.InsecureSkipVerify, - serverName: rCfg.ServerName, +func initCertConf(rCfg config.RabbitMQConfig) (*tls.Config, error) { + tlsConf := &tls.Config{ + InsecureSkipVerify: rCfg.InsecureSkipVerify, + ServerName: rCfg.ServerName, } caCert, err := os.ReadFile(rCfg.CACertPath) @@ -145,7 +174,7 @@ func initCertConf(rCfg config.RabbitMQConfig) (*RabbitMQCertConf, error) { if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { return nil, fmt.Errorf("failed to parse root certificate from %s", rCfg.CACertPath) } - certConf.caCertPool = caCertPool + tlsConf.RootCAs = caCertPool certPEM, err := os.ReadFile(rCfg.ClientCertPath) if err != nil { @@ -179,6 +208,6 @@ func initCertConf(rCfg config.RabbitMQConfig) (*RabbitMQCertConf, error) { return nil, fmt.Errorf("create x509 key pair failed: %w", err) } - certConf.clientCert = clientCert - return certConf, nil + tlsConf.Certificates = []tls.Certificate{clientCert} + return tlsConf, nil } diff --git a/real-time-data/real_time_data_computing.go b/real-time-data/real_time_data_computing.go index 88a2a8a..8b4fe45 100644 --- a/real-time-data/real_time_data_computing.go +++ b/real-time-data/real_time_data_computing.go @@ -165,7 +165,7 @@ func processCauseMap(data map[string]any) (map[string]any, error) { } } - if foundFloatKey == true { + if foundFloatKey { return causeResult, nil } From 1c385ee60d40606aa0fb4a344a698e383f662c3c Mon Sep 17 00:00:00 2001 From: douxu Date: Wed, 11 Feb 2026 16:43:42 +0800 Subject: [PATCH 13/43] optimize code of rabbitmq connection and event alarm struct --- mq/publish_event.go | 7 ++++--- mq/rabbitmq_init.go | 18 +++++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/mq/publish_event.go b/mq/publish_event.go index 7984741..2ea438a 100644 --- a/mq/publish_event.go +++ b/mq/publish_event.go @@ -19,7 +19,7 @@ const ( deadQueueName = "event-alarm-dead-letter-queue" ) -func initChannel(ctx context.Context) (*amqp.Channel, error) { +func initEventAlarmChannel(ctx context.Context) (*amqp.Channel, error) { var channel *amqp.Channel var err error @@ -70,8 +70,9 @@ func initChannel(ctx context.Context) (*amqp.Channel, error) { return channel, nil } -func pushEventToRabbitMQ(ctx context.Context, msgChan chan string) { - channel, err := initChannel(ctx) +// PushEventToRabbitMQ define func to push event alarm message to rabbitMQ +func PushEventToRabbitMQ(ctx context.Context, msgChan chan string) { + channel, err := initEventAlarmChannel(ctx) if err != nil { logger.Error(ctx, "initializing rabbitMQ channel failed", "error", err) return diff --git a/mq/rabbitmq_init.go b/mq/rabbitmq_init.go index 511547a..96c4fc0 100644 --- a/mq/rabbitmq_init.go +++ b/mq/rabbitmq_init.go @@ -7,7 +7,6 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "net/url" "os" "sync" "time" @@ -28,7 +27,7 @@ var ( type RabbitMQProxy struct { tlsConf *tls.Config conn *amqp.Connection - cancel context.CancelFunc // 增加这个用于停止重连协程 + cancel context.CancelFunc mu sync.Mutex } @@ -148,12 +147,17 @@ func CloseRabbitProxy() { } func generateRabbitMQURI(rCfg config.RabbitMQConfig) string { - user := url.QueryEscape(rCfg.User) - password := url.QueryEscape(rCfg.Password) + // TODO 考虑拆分用户名密码配置项，兼容不同认证方式 + // user := url.QueryEscape(rCfg.User) + // password := url.QueryEscape(rCfg.Password) - amqpURI := fmt.Sprintf("amqps://%s:%s@%s:%d/", - user, - password, + // amqpURI := fmt.Sprintf("amqps://%s:%s@%s:%d/", + // user, + // password, + // rCfg.Host, + // rCfg.Port, + // ) + amqpURI := fmt.Sprintf("amqps://%s:%d/", rCfg.Host, rCfg.Port, ) From 56b9999d6b961e7ba4157efcf283f6ebab829e54 Mon Sep 17 00:00:00 2001 From: douxu Date: Thu, 12 Feb 2026 17:09:08 +0800 Subject: [PATCH 14/43] add constants varibale of power system events --- constants/event.go | 63 ++++++++++++++++++++++++------------- constants/telemetry_imit.go | 31 ++++++++++++++++++ 2 files changed, 72 insertions(+), 22 deletions(-) create mode 100644 constants/telemetry_imit.go diff --git a/constants/event.go b/constants/event.go index 67bb63a..e3c3560 100644 --- a/constants/event.go +++ b/constants/event.go @@ -1,31 +1,50 @@ // Package constants define constant variable package constants -const ( - // TIBreachTriggerType define out of bounds type constant - TIBreachTriggerType = "trigger" -) +// EvenvtType define event type +type EvenvtType int const ( - // TelemetryUpLimit define telemetry upper limit - TelemetryUpLimit = "up" - // TelemetryUpUpLimit define telemetry upper upper limit - TelemetryUpUpLimit = "upup" - - // TelemetryDownLimit define telemetry limit - TelemetryDownLimit = "down" - // TelemetryDownDownLimit define telemetry lower lower limit - TelemetryDownDownLimit = "downdown" + // EventGeneralHard define gereral hard event type + EventGeneralHard EvenvtType = iota + // EventGeneralPlatformSoft define gereral platform soft event type + EventGeneralPlatformSoft + // EventGeneralApplicationSoft define gereral application soft event type + EventGeneralApplicationSoft + // EventWarnHard define warn hard event type + EventWarnHard + // EventWarnPlatformSoft define warn platform soft event type + EventWarnPlatformSoft + // EventWarnApplicationSoft define warn application soft event type + EventWarnApplicationSoft + // EventCriticalHard define critical hard event type + EventCriticalHard + // EventCriticalPlatformSoft define critical platform soft event type + EventCriticalPlatformSoft + // EventCriticalApplicationSoft define critical application soft event type + EventCriticalApplicationSoft ) +// IsGeneral define fucn to check event type is general +func IsGeneral(eventType EvenvtType) bool { + return eventType < 3 +} + +// IsWarning define fucn to check event type is warn +func IsWarning(eventType EvenvtType) bool { + return eventType >= 3 && eventType <= 5 +} + +// IsCritical define fucn to check event type is critical +func IsCritical(eventType EvenvtType) bool { + return eventType >= 6 +} + const ( - // TelesignalRaising define telesignal raising edge - TelesignalRaising = "raising" - // TelesignalFalling define telesignal falling edge - TelesignalFalling = "falling" -) - -const ( - // MinBreachCount define min breach count of real time data - MinBreachCount = 10 + // EventFromStation define event from station type + EventFromStation = "station" + // EventFromPlatform define event from platform type + EventFromPlatform = "platform" + // EventFromOthers define event from others type + EventFromOthers = "others" ) diff --git a/constants/telemetry_imit.go b/constants/telemetry_imit.go new file mode 100644 index 0000000..67bb63a --- /dev/null +++ b/constants/telemetry_imit.go @@ -0,0 +1,31 @@ +// Package constants define constant variable +package constants + +const ( + // TIBreachTriggerType define out of bounds type constant + TIBreachTriggerType = "trigger" +) + +const ( + // TelemetryUpLimit define telemetry upper limit + TelemetryUpLimit = "up" + // TelemetryUpUpLimit define telemetry upper upper limit + TelemetryUpUpLimit = "upup" + + // TelemetryDownLimit define telemetry limit + TelemetryDownLimit = "down" + // TelemetryDownDownLimit define telemetry lower lower limit + TelemetryDownDownLimit = "downdown" +) + +const ( + // TelesignalRaising define telesignal raising edge + TelesignalRaising = "raising" + // TelesignalFalling define telesignal falling edge + TelesignalFalling = "falling" +) + +const ( + // MinBreachCount define min breach count of real time data + MinBreachCount = 10 +) From 6c9da6fcd4c475d3243cd427ef1cd8f3926863ad Mon Sep 17 00:00:00 2001 From: douxu Date: Tue, 24 Feb 2026 17:08:48 +0800 Subject: [PATCH 15/43] init event struct with option mode --- alert/event.go | 23 ++------------ alert/event_options.go | 39 ++++++++++++++++++++++++ alert/gen_event.go | 68 ++++++++++++++++++++++++++++++++++++++++++ constants/event.go | 15 ++++++++++ 4 files changed, 124 insertions(+), 21 deletions(-) create mode 100644 alert/event_options.go create mode 100644 alert/gen_event.go diff --git a/alert/event.go b/alert/event.go index f580489..ca2ea81 100644 --- a/alert/event.go +++ b/alert/event.go @@ -4,7 +4,7 @@ package alert // EventRecord define struct for CIM event record type EventRecord struct { // 事件名称 - Event string `json:"event"` + EventName string `json:"event"` // 事件唯一标识符 EventUUID string `json:"event_uuid"` // 事件类型 @@ -28,7 +28,7 @@ type EventRecord struct { // 操作历史记录 (CIM ActivityRecord) Operations []OperationRecord `json:"operations"` // 子站告警原始数据 (CIM Alarm 数据) - Alarm map[string]any `json:"alarm,omitempty"` + Origin map[string]any `json:"origin,omitempty"` } // OperationRecord 描述对事件的操作记录，如确认(acknowledgment)等 @@ -37,22 +37,3 @@ type OperationRecord struct { Op string `json:"op"` // 操作人/操作账号标识 TS int64 `json:"ts"` // 操作发生的毫秒时间戳 } - -// 定义事件类型常量，便于逻辑判断 -const ( - TypeGeneralHard = 0 - TypeGeneralPlatformSoft = 1 - TypeGeneralApplicationSoft = 2 - TypeWarnHard = 3 - TypeWarnPlatformSoft = 4 - TypeWarnApplicationSoft = 5 - TypeCriticalHard = 6 - TypeCriticalPlatformSoft = 7 - TypeCriticalApplicationSoft = 8 -) - -const ( - FromStation = "station" - FromPlatform = "platform" - FromMSA = "msa" -) diff --git a/alert/event_options.go b/alert/event_options.go new file mode 100644 index 0000000..e114222 --- /dev/null +++ b/alert/event_options.go @@ -0,0 +1,39 @@ +// Package alert define alert event struct of modelRT project +package alert + +// EventOption 定义选项函数的类型 +type EventOption func(*EventRecord) + +// WithCondition 设置事件场景描述 +func WithCondition(cond map[string]any) EventOption { + return func(e *EventRecord) { + if cond != nil { + e.Condition = cond + } + } +} + +// WithSubscriptions 设置订阅信息 +func WithSubscriptions(subs []any) EventOption { + return func(e *EventRecord) { + if subs != nil { + e.AttachedSubscriptions = subs + } + } +} + +// WithOperations 设置操作记录 +func WithOperations(ops []OperationRecord) EventOption { + return func(e *EventRecord) { + if ops != nil { + e.Operations = ops + } + } +} + +// WithCategory 设置可选分类 +func WithCategory(cat string) EventOption { + return func(e *EventRecord) { + e.Category = cat + } +} diff --git a/alert/gen_event.go b/alert/gen_event.go new file mode 100644 index 0000000..d8b467e --- /dev/null +++ b/alert/gen_event.go @@ -0,0 +1,68 @@ +// Package alert define alert event struct of modelRT project +package alert + +import ( + "fmt" + "time" + + "modelRT/constants" + + "github.com/gofrs/uuid" +) + +// NewPlatformEventRecord define func to create a new platform event record with common fields initialized +func NewPlatformEventRecord(eventType int, priority int, eventName string, opts ...EventOption) (*EventRecord, error) { + u, err := uuid.NewV4() + if err != nil { + return nil, fmt.Errorf("failed to generate UUID: %w", err) + } + + record := &EventRecord{ + EventName: eventName, + EventUUID: u.String(), + Type: eventType, + Priority: priority, + Status: 1, + From: constants.EventFromPlatform, + Timestamp: time.Now().UnixNano() / int64(time.Millisecond), + Condition: make(map[string]any), + AttachedSubscriptions: make([]any, 0), + Operations: make([]OperationRecord, 0), + } + + for _, opt := range opts { + opt(record) + } + + return record, nil +} + +// NewGeneralPlatformSoftRecord define func to create a new general platform software event record +func NewGeneralPlatformSoftRecord(name string, opts ...EventOption) (*EventRecord, error) { + return NewPlatformEventRecord(int(constants.EventGeneralPlatformSoft), 0, name, opts...) +} + +// NewGeneralApplicationSoftRecord define func to create a new general application software event record +func NewGeneralApplicationSoftRecord(name string, opts ...EventOption) (*EventRecord, error) { + return NewPlatformEventRecord(int(constants.EventGeneralApplicationSoft), 0, name, opts...) +} + +// NewWarnPlatformSoftRecord define func to create a new warning platform software event record +func NewWarnPlatformSoftRecord(name string, opts ...EventOption) (*EventRecord, error) { + return NewPlatformEventRecord(int(constants.EventWarnPlatformSoft), 3, name, opts...) +} + +// NewWarnApplicationSoftRecord define func to create a new warning application software event record +func NewWarnApplicationSoftRecord(name string, opts ...EventOption) (*EventRecord, error) { + return NewPlatformEventRecord(int(constants.EventWarnApplicationSoft), 3, name, opts...) +} + +// NewCriticalPlatformSoftRecord define func to create a new critical platform software event record +func NewCriticalPlatformSoftRecord(name string, opts ...EventOption) (*EventRecord, error) { + return NewPlatformEventRecord(int(constants.EventCriticalPlatformSoft), 6, name, opts...) +} + +// NewCriticalApplicationSoftRecord define func to create a new critical application software event record +func NewCriticalApplicationSoftRecord(name string, opts ...EventOption) (*EventRecord, error) { + return NewPlatformEventRecord(int(constants.EventCriticalApplicationSoft), 6, name, opts...) +} diff --git a/constants/event.go b/constants/event.go index e3c3560..c9bc90d 100644 --- a/constants/event.go +++ b/constants/event.go @@ -48,3 +48,18 @@ const ( // EventFromOthers define event from others type EventFromOthers = "others" ) + +const ( + // EventStatusHappended define status for event record when event just happened, no data attached yet + EventStatusHappended = iota + // EventStatusDataAttached define status for event record when event just happened, data attached already + EventStatusDataAttached + // EventStatusReported define status for event record when event reported to CIM, no matter it's successful or failed + EventStatusReported + // EventStatusConfirmed define status for event record when event confirmed by CIM, no matter it's successful or failed + EventStatusConfirmed + // EventStatusPersisted define status for event record when event persisted in database, no matter it's successful or failed + EventStatusPersisted + // EventStatusClosed define status for event record when event closed, no matter it's successful or failed + EventStatusClosed +) From 2ececc38d90ca0ad66629e88bf78f17b5324a585 Mon Sep 17 00:00:00 2001 From: douxu Date: Wed, 25 Feb 2026 17:14:25 +0800 Subject: [PATCH 16/43] optimzie code organization structure of rabbitmq event --- alert/event_options.go | 39 ---- handler/alert_event_query.go | 2 +- handler/history_data_query.go | 2 +- main.go | 4 +- mq/publish_event.go | 7 + pool/concurrency_anchor_parse.go | 2 +- {alert => real-time-data/alert}/init.go | 0 real-time-data/compute_analyzer.go | 13 +- {alert => real-time-data/event}/event.go | 4 +- real-time-data/event/event_handlers.go | 38 ++-- real-time-data/event/event_options.go | 46 +++++ {alert => real-time-data/event}/gen_event.go | 4 +- real-time-data/real_time_data_computing.go | 178 ------------------- 13 files changed, 96 insertions(+), 243 deletions(-) delete mode 100644 alert/event_options.go rename {alert => real-time-data/alert}/init.go (100%) rename {alert => real-time-data/event}/event.go (94%) create mode 100644 real-time-data/event/event_options.go rename {alert => real-time-data/event}/gen_event.go (97%) diff --git a/alert/event_options.go b/alert/event_options.go deleted file mode 100644 index e114222..0000000 --- a/alert/event_options.go +++ /dev/null @@ -1,39 +0,0 @@ -// Package alert define alert event struct of modelRT project -package alert - -// EventOption 定义选项函数的类型 -type EventOption func(*EventRecord) - -// WithCondition 设置事件场景描述 -func WithCondition(cond map[string]any) EventOption { - return func(e *EventRecord) { - if cond != nil { - e.Condition = cond - } - } -} - -// WithSubscriptions 设置订阅信息 -func WithSubscriptions(subs []any) EventOption { - return func(e *EventRecord) { - if subs != nil { - e.AttachedSubscriptions = subs - } - } -} - -// WithOperations 设置操作记录 -func WithOperations(ops []OperationRecord) EventOption { - return func(e *EventRecord) { - if ops != nil { - e.Operations = ops - } - } -} - -// WithCategory 设置可选分类 -func WithCategory(cat string) EventOption { - return func(e *EventRecord) { - e.Category = cat - } -} diff --git a/handler/alert_event_query.go b/handler/alert_event_query.go index 58e1613..b49d954 100644 --- a/handler/alert_event_query.go +++ b/handler/alert_event_query.go @@ -5,10 +5,10 @@ import ( "net/http" "strconv" - "modelRT/alert" "modelRT/constants" "modelRT/logger" "modelRT/network" + "modelRT/real-time-data/alert" "github.com/gin-gonic/gin" ) diff --git a/handler/history_data_query.go b/handler/history_data_query.go index 294f4b9..d8718e1 100644 --- a/handler/history_data_query.go +++ b/handler/history_data_query.go @@ -6,10 +6,10 @@ import ( "net/http" "strconv" - "modelRT/alert" "modelRT/constants" "modelRT/logger" "modelRT/network" + "modelRT/real-time-data/alert" "github.com/gin-gonic/gin" ) diff --git a/main.go b/main.go index 8cc8a76..2c1c4b8 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,6 @@ import ( "path/filepath" "syscall" - "modelRT/alert" "modelRT/config" "modelRT/constants" "modelRT/database" @@ -22,6 +21,7 @@ import ( "modelRT/model" "modelRT/mq" "modelRT/pool" + "modelRT/real-time-data/alert" "modelRT/router" "modelRT/util" @@ -152,6 +152,8 @@ func main() { // init rabbitmq connection mq.InitRabbitProxy(ctx, modelRTConfig.RabbitMQConfig) + // async push event to rabbitMQ + go mq.PushEventToRabbitMQ(ctx, mq.MsgChan) postgresDBClient.Transaction(func(tx *gorm.DB) error { // load circuit diagram from postgres diff --git a/mq/publish_event.go b/mq/publish_event.go index 2ea438a..9270e93 100644 --- a/mq/publish_event.go +++ b/mq/publish_event.go @@ -10,6 +10,13 @@ import ( amqp "github.com/rabbitmq/amqp091-go" ) +// MsgChan define variable of channel to store messages that need to be sent to rabbitMQ +var MsgChan chan string + +func init() { + MsgChan = make(chan string, 10000) +} + const ( routingKey = "event-alarm-routing-key" exchangeName = "event-alarm-exchange" diff --git a/pool/concurrency_anchor_parse.go b/pool/concurrency_anchor_parse.go index 443f1a7..85fa225 100644 --- a/pool/concurrency_anchor_parse.go +++ b/pool/concurrency_anchor_parse.go @@ -5,11 +5,11 @@ import ( "fmt" "time" - "modelRT/alert" "modelRT/config" "modelRT/constants" "modelRT/diagram" "modelRT/logger" + "modelRT/real-time-data/alert" "github.com/panjf2000/ants/v2" ) diff --git a/alert/init.go b/real-time-data/alert/init.go similarity index 100% rename from alert/init.go rename to real-time-data/alert/init.go diff --git a/real-time-data/compute_analyzer.go b/real-time-data/compute_analyzer.go index 1525734..62a7ecc 100644 --- a/real-time-data/compute_analyzer.go +++ b/real-time-data/compute_analyzer.go @@ -124,9 +124,15 @@ func analyzeTEDataLogic(ctx context.Context, conf *ComputeConfig, thresholds teE } } } - + // TODO 记录触发trigger的type即firstValueBreachType，并考虑如何融合更多错误信息 if eventTriggered { command, content := genTEEventCommandAndContent(ctx, conf.Action) + // TODO 生成 condition 并考虑如何放入 event 中 + event.WithCondition(conf.Cause) + // TODO 生成 result 并考虑如何放入 event 中 + event.WithResult(map[string]any{"real_time_values": realTimeValues}) + // TODO 生成 operations并考虑如何放入 event 中 + event.WithOperations(nil) // TODO 考虑 content 是否可以为空,先期不允许 if command == "" || content == "" { logger.Error(ctx, "generate telemetry evnet command or content failed", "action", conf.Action, "command", command, "content", content) @@ -211,11 +217,12 @@ func parseTIThresholds(cause map[string]any) (tiEventThresholds, error) { // getTIBreachType define func to determine which type of out-of-limit the telesignal real time data belongs to func getTIBreachType(currentValue float64, previousValue float64, t tiEventThresholds) string { - if t.edge == constants.TelesignalRaising { + switch t.edge { + case constants.TelesignalRaising: if previousValue == 0.0 && currentValue == 1.0 { return constants.TIBreachTriggerType } - } else if t.edge == constants.TelesignalFalling { + case constants.TelesignalFalling: if previousValue == 1.0 && currentValue == 0.0 { return constants.TIBreachTriggerType } diff --git a/alert/event.go b/real-time-data/event/event.go similarity index 94% rename from alert/event.go rename to real-time-data/event/event.go index ca2ea81..6740251 100644 --- a/alert/event.go +++ b/real-time-data/event/event.go @@ -1,5 +1,5 @@ -// Package alert define alert event struct of modelRT project -package alert +// Package event define real time data evnet operation functions +package event // EventRecord define struct for CIM event record type EventRecord struct { diff --git a/real-time-data/event/event_handlers.go b/real-time-data/event/event_handlers.go index a161a7d..8370c0c 100644 --- a/real-time-data/event/event_handlers.go +++ b/real-time-data/event/event_handlers.go @@ -3,11 +3,13 @@ package event import ( "context" + "fmt" "modelRT/logger" + "modelRT/mq" ) -type actionHandler func(ctx context.Context, content string) error +type actionHandler func(ctx context.Context, content string, ops ...EventOption) error // actionDispatchMap define variable to store all action handler into map var actionDispatchMap = map[string]actionHandler{ @@ -19,13 +21,13 @@ var actionDispatchMap = map[string]actionHandler{ } // TriggerEventAction define func to trigger event by action in compute config -func TriggerEventAction(ctx context.Context, command string, content string) { +func TriggerEventAction(ctx context.Context, command string, content string, ops ...EventOption) { handler, exists := actionDispatchMap[command] if !exists { logger.Error(ctx, "unknown action command", "command", command) return } - err := handler(ctx, content) + err := handler(ctx, content, ops...) if err != nil { logger.Error(ctx, "action handler failed", "command", command, "content", content, "error", err) return @@ -33,7 +35,7 @@ func TriggerEventAction(ctx context.Context, command string, content string) { logger.Info(ctx, "action handler success", "command", command, "content", content) } -func handleInfoAction(ctx context.Context, content string) error { +func handleInfoAction(ctx context.Context, content string, ops ...EventOption) error { // 实际执行发送警告、记录日志等操作 actionParams := content // ... logic to send info level event using actionParams ... @@ -41,23 +43,29 @@ func handleInfoAction(ctx context.Context, content string) error { return nil } -func handleWarningAction(ctx context.Context, content string) error { - // 实际执行发送警告、记录日志等操作 - actionParams := content - // ... logic to send warning level event using actionParams ... - logger.Warn(ctx, "trigger warning event", "message", actionParams) +func handleWarningAction(ctx context.Context, eventName string, ops ...EventOption) error { + eventRecord, err := NewWarnPlatformSoftRecord(eventName, ops...) + if err != nil { + logger.Error(ctx, "failed to create event record", "error", err) + return err + } + mq.MsgChan <- fmt.Sprintf("Generated event record: %+v", eventRecord) + logger.Info(ctx, "trigger warning event", "event_name", eventName) return nil } -func handleErrorAction(ctx context.Context, content string) error { - // 实际执行发送警告、记录日志等操作 +func handleErrorAction(ctx context.Context, content string, ops ...EventOption) error { actionParams := content - // ... logic to send error level event using actionParams ... - logger.Warn(ctx, "trigger error event", "message", actionParams) + eventRecord, err := NewCriticalPlatformSoftRecord("ErrorEvent", WithCondition(map[string]any{"message": actionParams})) + if err != nil { + logger.Error(ctx, "failed to create event record", "error", err) + return err + } + mq.MsgChan <- fmt.Sprintf("Generated event record: %+v", eventRecord) return nil } -func handleCriticalAction(ctx context.Context, content string) error { +func handleCriticalAction(ctx context.Context, content string, ops ...EventOption) error { // 实际执行发送警告、记录日志等操作 actionParams := content // ... logic to send critical level event using actionParams ... @@ -65,7 +73,7 @@ func handleCriticalAction(ctx context.Context, content string) error { return nil } -func handleExceptionAction(ctx context.Context, content string) error { +func handleExceptionAction(ctx context.Context, content string, ops ...EventOption) error { // 实际执行发送警告、记录日志等操作 actionParams := content // ... logic to send except level event using actionParams ... diff --git a/real-time-data/event/event_options.go b/real-time-data/event/event_options.go new file mode 100644 index 0000000..fda0c59 --- /dev/null +++ b/real-time-data/event/event_options.go @@ -0,0 +1,46 @@ +// Package event define real time data evnet operation functions +package event + +// EventOption define option function type for event record creation +type EventOption func(*EventRecord) + +// WithCondition define option function to set event condition description +func WithCondition(cond map[string]any) EventOption { + return func(e *EventRecord) { + if cond != nil { + e.Condition = cond + } + } +} + +// WithSubscriptions define option function to set event attached subscription information +func WithSubscriptions(subs []any) EventOption { + return func(e *EventRecord) { + if subs != nil { + e.AttachedSubscriptions = subs + } + } +} + +// WithOperations define option function to set event operation records +func WithOperations(ops []OperationRecord) EventOption { + return func(e *EventRecord) { + if ops != nil { + e.Operations = ops + } + } +} + +// WithCategory define option function to set event category +func WithCategory(cat string) EventOption { + return func(e *EventRecord) { + e.Category = cat + } +} + +// WithResult define option function to set event analysis result +func WithResult(result map[string]any) EventOption { + return func(e *EventRecord) { + e.Result = result + } +} diff --git a/alert/gen_event.go b/real-time-data/event/gen_event.go similarity index 97% rename from alert/gen_event.go rename to real-time-data/event/gen_event.go index d8b467e..7b4b094 100644 --- a/alert/gen_event.go +++ b/real-time-data/event/gen_event.go @@ -1,5 +1,5 @@ -// Package alert define alert event struct of modelRT project -package alert +// Package event define real time data evnet operation functions +package event import ( "fmt" diff --git a/real-time-data/real_time_data_computing.go b/real-time-data/real_time_data_computing.go index 8b4fe45..b41446d 100644 --- a/real-time-data/real_time_data_computing.go +++ b/real-time-data/real_time_data_computing.go @@ -227,181 +227,3 @@ func continuousComputation(ctx context.Context, conf *ComputeConfig) { } } } - -// // ReceiveChan define func to real time data receive and process -// func ReceiveChan(ctx context.Context, consumerConfig *kafka.ConfigMap, topics []string, duration float32) { -// consumer, err := kafka.NewConsumer(consumerConfig) -// if err != nil { -// logger.Error(ctx, "create kafka consumer failed", "error", err) -// return -// } -// defer consumer.Close() - -// err = consumer.SubscribeTopics(topics, nil) -// if err != nil { -// logger.Error(ctx, "subscribe kafka topics failed", "topic", topics, "error", err) -// return -// } - -// batchSize := 100 -// batchTimeout := util.SecondsToDuration(duration) -// messages := make([]*kafka.Message, 0, batchSize) -// lastCommit := time.Now() -// logger.Info(ctx, "start consuming from kafka", "topic", topics) -// for { -// select { -// case <-ctx.Done(): -// logger.Info(ctx, "stop real time data computing by context cancel") -// return -// case realTimeData := <-RealTimeDataChan: -// componentUUID := realTimeData.PayLoad.ComponentUUID -// component, err := diagram.GetComponentMap(componentUUID) -// if err != nil { -// logger.Error(ctx, "query component info from diagram map by componet id failed", "component_uuid", componentUUID, "error", err) -// continue -// } - -// componentType := component.Type -// if componentType != constants.DemoType { -// logger.Error(ctx, "can not process real time data of component type not equal DemoType", "component_uuid", componentUUID) -// continue -// } - -// var anchorName string -// var compareValUpperLimit, compareValLowerLimit float64 -// var anchorRealTimeData []float64 -// var calculateFunc func(archorValue float64, args ...float64) float64 - -// // calculateFunc, params := config.SelectAnchorCalculateFuncAndParams(componentType, anchorName, componentData) - -// for _, param := range realTimeData.PayLoad.Values { -// anchorRealTimeData = append(anchorRealTimeData, param.Value) -// } - -// anchorConfig := config.AnchorParamConfig{ -// AnchorParamBaseConfig: config.AnchorParamBaseConfig{ -// ComponentUUID: componentUUID, -// AnchorName: anchorName, -// CompareValUpperLimit: compareValUpperLimit, -// CompareValLowerLimit: compareValLowerLimit, -// AnchorRealTimeData: anchorRealTimeData, -// }, -// CalculateFunc: calculateFunc, -// CalculateParams: []float64{}, -// } -// anchorChan, err := pool.GetAnchorParamChan(ctx, componentUUID) -// if err != nil { -// logger.Error(ctx, "get anchor param chan failed", "component_uuid", componentUUID, "error", err) -// continue -// } -// anchorChan <- anchorConfig -// default: -// msg, err := consumer.ReadMessage(batchTimeout) -// if err != nil { -// if err.(kafka.Error).Code() == kafka.ErrTimedOut { -// // process accumulated messages when timeout -// if len(messages) > 0 { -// processMessageBatch(ctx, messages) -// consumer.Commit() -// messages = messages[:0] -// } -// continue -// } -// logger.Error(ctx, "read message from kafka failed", "error", err, "msg", msg) -// continue -// } - -// messages = append(messages, msg) -// // process messages when batch size or timeout period is reached -// if len(messages) >= batchSize || time.Since(lastCommit) >= batchTimeout { -// processMessageBatch(ctx, messages) -// consumer.Commit() -// messages = messages[:0] -// lastCommit = time.Now() -// } -// } -// } -// } - -// type realTimeDataPayload struct { -// ComponentUUID string -// Values []float64 -// } - -// type realTimeData struct { -// Payload realTimeDataPayload -// } - -// func parseKafkaMessage(msgValue []byte) (*realTimeData, error) { -// var realTimeData realTimeData -// err := json.Unmarshal(msgValue, &realTimeData) -// if err != nil { -// return nil, fmt.Errorf("unmarshal real time data failed: %w", err) -// } -// return &realTimeData, nil -// } - -// func processRealTimeData(ctx context.Context, realTimeData *realTimeData) { -// componentUUID := realTimeData.Payload.ComponentUUID -// component, err := diagram.GetComponentMap(componentUUID) -// if err != nil { -// logger.Error(ctx, "query component info from diagram map by component id failed", -// "component_uuid", componentUUID, "error", err) -// return -// } - -// componentType := component.Type -// if componentType != constants.DemoType { -// logger.Error(ctx, "can not process real time data of component type not equal DemoType", -// "component_uuid", componentUUID) -// return -// } - -// var anchorName string -// var compareValUpperLimit, compareValLowerLimit float64 -// var anchorRealTimeData []float64 -// var calculateFunc func(archorValue float64, args ...float64) float64 - -// for _, param := range realTimeData.Payload.Values { -// anchorRealTimeData = append(anchorRealTimeData, param) -// } - -// anchorConfig := config.AnchorParamConfig{ -// AnchorParamBaseConfig: config.AnchorParamBaseConfig{ -// ComponentUUID: componentUUID, -// AnchorName: anchorName, -// CompareValUpperLimit: compareValUpperLimit, -// CompareValLowerLimit: compareValLowerLimit, -// AnchorRealTimeData: anchorRealTimeData, -// }, -// CalculateFunc: calculateFunc, -// CalculateParams: []float64{}, -// } - -// anchorChan, err := pool.GetAnchorParamChan(ctx, componentUUID) -// if err != nil { -// logger.Error(ctx, "get anchor param chan failed", -// "component_uuid", componentUUID, "error", err) -// return -// } - -// select { -// case anchorChan <- anchorConfig: -// case <-ctx.Done(): -// logger.Info(ctx, "context done while sending to anchor chan") -// case <-time.After(5 * time.Second): -// logger.Error(ctx, "timeout sending to anchor chan", "component_uuid", componentUUID) -// } -// } - -// // processMessageBatch define func to bathc process kafka message -// func processMessageBatch(ctx context.Context, messages []*kafka.Message) { -// for _, msg := range messages { -// realTimeData, err := parseKafkaMessage(msg.Value) -// if err != nil { -// logger.Error(ctx, "parse kafka message failed", "error", err, "msg", msg) -// continue -// } -// go processRealTimeData(ctx, realTimeData) -// } -// } From f6bb3fb985732977f66a356c3212ccba28c11764 Mon Sep 17 00:00:00 2001 From: douxu Date: Thu, 26 Feb 2026 16:48:12 +0800 Subject: [PATCH 17/43] optimize code of push event to rabbitmq --- main.go | 2 +- mq/publish_event.go | 8 +- real-time-data/compute_analyzer.go | 93 ++++++++++--------- real-time-data/event/event_handlers.go | 51 ++++++---- real-time-data/event/event_options.go | 39 ++++++++ ...real_time_data_up_down_limit_computing.go} | 6 +- 6 files changed, 132 insertions(+), 67 deletions(-) rename real-time-data/{real_time_data_computing.go => real_time_data_up_down_limit_computing.go} (94%) diff --git a/main.go b/main.go index 2c1c4b8..486a7ef 100644 --- a/main.go +++ b/main.go @@ -204,7 +204,7 @@ func main() { logger.Error(ctx, "load topologic info from postgres failed", "error", err) panic(err) } - go realtimedata.StartRealTimeDataComputing(ctx, allMeasurement) + go realtimedata.StartComputingRealTimeDataLimit(ctx, allMeasurement) tree, err := database.QueryTopologicFromDB(ctx, tx) if err != nil { diff --git a/mq/publish_event.go b/mq/publish_event.go index 9270e93..7b0ee6f 100644 --- a/mq/publish_event.go +++ b/mq/publish_event.go @@ -11,10 +11,10 @@ import ( ) // MsgChan define variable of channel to store messages that need to be sent to rabbitMQ -var MsgChan chan string +var MsgChan chan []byte func init() { - MsgChan = make(chan string, 10000) + MsgChan = make(chan []byte, 10000) } const ( @@ -78,7 +78,7 @@ func initEventAlarmChannel(ctx context.Context) (*amqp.Channel, error) { } // PushEventToRabbitMQ define func to push event alarm message to rabbitMQ -func PushEventToRabbitMQ(ctx context.Context, msgChan chan string) { +func PushEventToRabbitMQ(ctx context.Context, msgChan chan []byte) { channel, err := initEventAlarmChannel(ctx) if err != nil { logger.Error(ctx, "initializing rabbitMQ channel failed", "error", err) @@ -125,7 +125,7 @@ func PushEventToRabbitMQ(ctx context.Context, msgChan chan string) { false, // immediate amqp.Publishing{ ContentType: "text/plain", - Body: []byte(msg), + Body: msg, }) cancel() diff --git a/real-time-data/compute_analyzer.go b/real-time-data/compute_analyzer.go index 62a7ecc..c1ae86a 100644 --- a/real-time-data/compute_analyzer.go +++ b/real-time-data/compute_analyzer.go @@ -26,6 +26,13 @@ type teEventThresholds struct { isFloatCause bool } +type teBreachTrigger struct { + breachType string + triggered bool + triggeredValues []float64 + eventOpts []event.EventOption +} + // parseTEThresholds define func to parse telemetry thresholds by casue map func parseTEThresholds(cause map[string]any) (teEventThresholds, error) { t := teEventThresholds{} @@ -84,66 +91,69 @@ func (t *TEAnalyzer) AnalyzeAndTriggerEvent(ctx context.Context, conf *ComputeCo // analyzeTEDataLogic define func to processing telemetry data and event triggering func analyzeTEDataLogic(ctx context.Context, conf *ComputeConfig, thresholds teEventThresholds, realTimeValues []float64) { windowSize := conf.minBreachCount - if windowSize <= 0 { - logger.Error(ctx, "variable minBreachCount is invalid or zero, analysis skipped", "minBreachCount", windowSize) + dataLen := len(realTimeValues) + if dataLen < windowSize || windowSize <= 0 { return } - // mark whether any events have been triggered in this batch - var eventTriggered bool - breachTriggers := map[string]bool{ - "up": false, "upup": false, "down": false, "downdown": false, + statusArray := make([]string, dataLen) + for i, val := range realTimeValues { + statusArray[i] = getTEBreachType(val, thresholds) } - // implement slide window to determine breach counts - for i := 0; i <= len(realTimeValues)-windowSize; i++ { - window := realTimeValues[i : i+windowSize] - firstValueBreachType := getTEBreachType(window[0], thresholds) + breachTriggers := make(map[string]teBreachTrigger) + for i := 0; i <= dataLen-windowSize; i++ { + firstBreachType := statusArray[i] - if firstValueBreachType == "" { + // if the first value in the window does not breach, skip this window directly + if firstBreachType == "" { continue } allMatch := true for j := 1; j < windowSize; j++ { - currentValueBreachType := getTEBreachType(window[j], thresholds) - if currentValueBreachType != firstValueBreachType { + if statusArray[i+j] != firstBreachType { allMatch = false break } } if allMatch { + triggerValues := realTimeValues[i : i+windowSize] // in the case of a continuous sequence of out-of-limit events, check whether this type of event has already been triggered in the current batch of data - if !breachTriggers[firstValueBreachType] { - // trigger event - logger.Warn(ctx, "event triggered by sliding window", "breach_type", firstValueBreachType, "value", window[windowSize-1]) + _, exists := breachTriggers[firstBreachType] + if !exists { + logger.Warn(ctx, "event triggered by sliding window", + "breach_type", firstBreachType, + "trigger_values", triggerValues) - breachTriggers[firstValueBreachType] = true - eventTriggered = true + // build Options + opts := []event.EventOption{ + event.WithConditionValue(triggerValues, conf.Cause), + event.WithTEAnalysisResult(firstBreachType), + // TODO 生成 operations并考虑如何放入 event 中 + // event.WithOperations(nil) + } + breachTriggers[firstBreachType] = teBreachTrigger{ + breachType: firstBreachType, + triggered: false, + triggeredValues: triggerValues, + eventOpts: opts, + } } } } - // TODO 记录触发trigger的type即firstValueBreachType，并考虑如何融合更多错误信息 - if eventTriggered { - command, content := genTEEventCommandAndContent(ctx, conf.Action) - // TODO 生成 condition 并考虑如何放入 event 中 - event.WithCondition(conf.Cause) - // TODO 生成 result 并考虑如何放入 event 中 - event.WithResult(map[string]any{"real_time_values": realTimeValues}) - // TODO 生成 operations并考虑如何放入 event 中 - event.WithOperations(nil) - // TODO 考虑 content 是否可以为空,先期不允许 - if command == "" || content == "" { - logger.Error(ctx, "generate telemetry evnet command or content failed", "action", conf.Action, "command", command, "content", content) - return - } - event.TriggerEventAction(ctx, command, content) - return + + for breachType, trigger := range breachTriggers { + // trigger Action + command, mainBody := genTEEventCommandAndMainBody(ctx, conf.Action) + eventName := fmt.Sprintf("telemetry_%s_%s_Breach_Event", mainBody, breachType) + event.TriggerEventAction(ctx, command, eventName, trigger.eventOpts...) + } } -func genTEEventCommandAndContent(ctx context.Context, action map[string]any) (command string, content string) { +func genTEEventCommandAndMainBody(ctx context.Context, action map[string]any) (command string, mainBody string) { cmdValue, exist := action["command"] if !exist { logger.Error(ctx, "can not find command variable into action map", "action", action) @@ -191,7 +201,7 @@ type tiEventThresholds struct { isFloatCause bool } -// parseTEThresholds define func to parse telesignal thresholds by casue map +// parseTIThresholds define func to parse telesignal thresholds by casue map func parseTIThresholds(cause map[string]any) (tiEventThresholds, error) { edgeKey := "edge" t := tiEventThresholds{ @@ -304,18 +314,17 @@ func analyzeTIDataLogic(ctx context.Context, conf *ComputeConfig, thresholds tiE } if eventTriggered { - command, content := genTIEventCommandAndContent(conf.Action) - // TODO 考虑 content 是否可以为空,先期不允许 - if command == "" || content == "" { - logger.Error(ctx, "generate telemetry evnet command or content failed", "action", conf.Action, "command", command, "content", content) + command, mainBody := genTIEventCommandAndMainBody(conf.Action) + if command == "" || mainBody == "" { + logger.Error(ctx, "generate telemetry evnet command or content failed", "action", conf.Action, "command", command, "main_body", mainBody) return } - event.TriggerEventAction(ctx, command, content) + event.TriggerEventAction(ctx, command, mainBody) return } } -func genTIEventCommandAndContent(action map[string]any) (command string, content string) { +func genTIEventCommandAndMainBody(action map[string]any) (command string, mainBody string) { cmdValue, exist := action["command"] if !exist { return "", "" diff --git a/real-time-data/event/event_handlers.go b/real-time-data/event/event_handlers.go index 8370c0c..841b06e 100644 --- a/real-time-data/event/event_handlers.go +++ b/real-time-data/event/event_handlers.go @@ -3,7 +3,7 @@ package event import ( "context" - "fmt" + "encoding/json" "modelRT/logger" "modelRT/mq" @@ -21,47 +21,64 @@ var actionDispatchMap = map[string]actionHandler{ } // TriggerEventAction define func to trigger event by action in compute config -func TriggerEventAction(ctx context.Context, command string, content string, ops ...EventOption) { +func TriggerEventAction(ctx context.Context, command string, eventName string, ops ...EventOption) { handler, exists := actionDispatchMap[command] if !exists { logger.Error(ctx, "unknown action command", "command", command) return } - err := handler(ctx, content, ops...) + err := handler(ctx, eventName, ops...) if err != nil { - logger.Error(ctx, "action handler failed", "command", command, "content", content, "error", err) + logger.Error(ctx, "action handler failed", "command", command, "event_name", eventName, "error", err) return } - logger.Info(ctx, "action handler success", "command", command, "content", content) + logger.Info(ctx, "action handler success", "command", command, "event_name", eventName) } -func handleInfoAction(ctx context.Context, content string, ops ...EventOption) error { - // 实际执行发送警告、记录日志等操作 - actionParams := content - // ... logic to send info level event using actionParams ... - logger.Warn(ctx, "trigger info event", "message", actionParams) +func handleInfoAction(ctx context.Context, eventName string, ops ...EventOption) error { + eventRecord, err := NewGeneralPlatformSoftRecord(eventName, ops...) + if err != nil { + logger.Error(ctx, "generate info event record failed", "error", err) + return err + } + recordBytes, err := json.Marshal(eventRecord) + if err != nil { + logger.Error(ctx, "marshal event record failed", "event_uuid", eventRecord.EventUUID, "error", err) + return err + } + mq.MsgChan <- recordBytes + logger.Info(ctx, "trigger info event", "event_name", eventName) return nil } func handleWarningAction(ctx context.Context, eventName string, ops ...EventOption) error { eventRecord, err := NewWarnPlatformSoftRecord(eventName, ops...) if err != nil { - logger.Error(ctx, "failed to create event record", "error", err) + logger.Error(ctx, "generate warning event record failed", "error", err) return err } - mq.MsgChan <- fmt.Sprintf("Generated event record: %+v", eventRecord) + recordBytes, err := json.Marshal(eventRecord) + if err != nil { + logger.Error(ctx, "marshal event record failed", "event_uuid", eventRecord.EventUUID, "error", err) + return err + } + mq.MsgChan <- recordBytes logger.Info(ctx, "trigger warning event", "event_name", eventName) return nil } -func handleErrorAction(ctx context.Context, content string, ops ...EventOption) error { - actionParams := content - eventRecord, err := NewCriticalPlatformSoftRecord("ErrorEvent", WithCondition(map[string]any{"message": actionParams})) +func handleErrorAction(ctx context.Context, eventName string, ops ...EventOption) error { + eventRecord, err := NewCriticalPlatformSoftRecord(eventName, ops...) if err != nil { - logger.Error(ctx, "failed to create event record", "error", err) + logger.Error(ctx, "generate error event record failed", "error", err) return err } - mq.MsgChan <- fmt.Sprintf("Generated event record: %+v", eventRecord) + recordBytes, err := json.Marshal(eventRecord) + if err != nil { + logger.Error(ctx, "marshal event record failed", "event_uuid", eventRecord.EventUUID, "error", err) + return err + } + mq.MsgChan <- recordBytes return nil } diff --git a/real-time-data/event/event_options.go b/real-time-data/event/event_options.go index fda0c59..38e7b45 100644 --- a/real-time-data/event/event_options.go +++ b/real-time-data/event/event_options.go @@ -1,6 +1,11 @@ // Package event define real time data evnet operation functions package event +import ( + "maps" + "strings" +) + // EventOption define option function type for event record creation type EventOption func(*EventRecord) @@ -44,3 +49,37 @@ func WithResult(result map[string]any) EventOption { e.Result = result } } + +func WithTEAnalysisResult(breachType string) EventOption { + return func(e *EventRecord) { + if e.Result == nil { + e.Result = make(map[string]any) + } + + description := "数据异常" + switch strings.ToLower(breachType) { + case "upup": + description = "超越上上限" + case "up": + description = "超越上限" + case "down": + description = "超越下限" + case "downdown": + description = "超越下下限" + } + + e.Result["analysis_desc"] = description + e.Result["breach_type"] = breachType + } +} + +// WithConditionValue define option function to set event condition with real time value and extra data +func WithConditionValue(realTimeValue []float64, extraData map[string]any) EventOption { + return func(e *EventRecord) { + if e.Condition == nil { + e.Condition = make(map[string]any) + } + e.Condition["real_time_value"] = realTimeValue + maps.Copy(e.Condition, extraData) + } +} diff --git a/real-time-data/real_time_data_computing.go b/real-time-data/real_time_data_up_down_limit_computing.go similarity index 94% rename from real-time-data/real_time_data_computing.go rename to real-time-data/real_time_data_up_down_limit_computing.go index b41446d..53acc8d 100644 --- a/real-time-data/real_time_data_computing.go +++ b/real-time-data/real_time_data_up_down_limit_computing.go @@ -27,8 +27,8 @@ func init() { globalComputeState = NewMeasComputeState() } -// StartRealTimeDataComputing define func to start real time data process goroutines by measurement info -func StartRealTimeDataComputing(ctx context.Context, measurements []orm.Measurement) { +// StartComputingRealTimeDataLimit define func to start compute real time data up or down limit process goroutines by measurement info +func StartComputingRealTimeDataLimit(ctx context.Context, measurements []orm.Measurement) { for _, measurement := range measurements { enableValue, exist := measurement.EventPlan["enable"] enable, ok := enableValue.(bool) @@ -57,7 +57,7 @@ func StartRealTimeDataComputing(ctx context.Context, measurements []orm.Measurem enrichedCtx := context.WithValue(ctx, constants.MeasurementUUIDKey, uuidStr) conf.StopGchan = make(chan struct{}) globalComputeState.Store(uuidStr, conf) - logger.Info(ctx, "starting real time data computing for measurement", "measurement_uuid", measurement.ComponentUUID) + logger.Info(ctx, "starting computing real time data limit for measurement", "measurement_uuid", measurement.ComponentUUID) go continuousComputation(enrichedCtx, conf) } } From 4b52e5f3c6298d6ce3673baf118b56e019274e17 Mon Sep 17 00:00:00 2001 From: douxu Date: Sat, 28 Feb 2026 17:38:33 +0800 Subject: [PATCH 18/43] optimize code of event record and push rabbitmq func --- constants/event.go | 18 ++++++++ main.go | 2 +- ...vent.go => publish_up_down_limit_event.go} | 42 ++++++++----------- real-time-data/compute_analyzer.go | 1 + 4 files changed, 37 insertions(+), 26 deletions(-) rename mq/{publish_event.go => publish_up_down_limit_event.go} (66%) diff --git a/constants/event.go b/constants/event.go index c9bc90d..435cb2b 100644 --- a/constants/event.go +++ b/constants/event.go @@ -63,3 +63,21 @@ const ( // EventStatusClosed define status for event record when event closed, no matter it's successful or failed EventStatusClosed ) + +const ( + // EventExchangeName define exchange name for event alarm message + EventExchangeName = "event-exchange" + // EventDeadExchangeName define dead letter exchange name for event alarm message + EventDeadExchangeName = "event-dead-letter-exchange" +) + +const ( + // EventUpDownRoutingKey define routing key for up or down limit event alarm message + EventUpDownRoutingKey = "event-up-down-routing-key" + // EventUpDownDeadRoutingKey define dead letter routing key for up or down limit event alarm message + EventUpDownDeadRoutingKey = "event-up-down-dead-letter-routing-key" + // EventUpDownQueueName define queue name for up or down limit event alarm message + EventUpDownQueueName = "event-up-down-queue" + // EventUpDownDeadQueueName define dead letter queue name for event alarm message + EventUpDownDeadQueueName = "event-dead-letter-queue" +) diff --git a/main.go b/main.go index 486a7ef..c718793 100644 --- a/main.go +++ b/main.go @@ -153,7 +153,7 @@ func main() { // init rabbitmq connection mq.InitRabbitProxy(ctx, modelRTConfig.RabbitMQConfig) // async push event to rabbitMQ - go mq.PushEventToRabbitMQ(ctx, mq.MsgChan) + go mq.PushUpDownLimitEventToRabbitMQ(ctx, mq.MsgChan) postgresDBClient.Transaction(func(tx *gorm.DB) error { // load circuit diagram from postgres diff --git a/mq/publish_event.go b/mq/publish_up_down_limit_event.go similarity index 66% rename from mq/publish_event.go rename to mq/publish_up_down_limit_event.go index 7b0ee6f..e1f0bd1 100644 --- a/mq/publish_event.go +++ b/mq/publish_up_down_limit_event.go @@ -5,6 +5,7 @@ import ( "context" "time" + "modelRT/constants" "modelRT/logger" amqp "github.com/rabbitmq/amqp091-go" @@ -17,16 +18,7 @@ func init() { MsgChan = make(chan []byte, 10000) } -const ( - routingKey = "event-alarm-routing-key" - exchangeName = "event-alarm-exchange" - queueName = "event-alarm-queue" - deadRoutingKey = "event-alarm-dead-letter-routing-key" - deadExchangeName = "event-alarm-dead-letter-exchange" - deadQueueName = "event-alarm-dead-letter-queue" -) - -func initEventAlarmChannel(ctx context.Context) (*amqp.Channel, error) { +func initUpDownLimitEventChannel(ctx context.Context) (*amqp.Channel, error) { var channel *amqp.Channel var err error @@ -35,22 +27,22 @@ func initEventAlarmChannel(ctx context.Context) (*amqp.Channel, error) { logger.Error(ctx, "open rabbitMQ server channel failed", "error", err) } - err = channel.ExchangeDeclare(deadExchangeName, "topic", true, false, false, false, nil) + err = channel.ExchangeDeclare(constants.EventDeadExchangeName, "topic", true, false, false, false, nil) if err != nil { logger.Error(ctx, "declare event dead letter exchange failed", "error", err) } - _, err = channel.QueueDeclare(deadQueueName, true, false, false, false, nil) + _, err = channel.QueueDeclare(constants.EventUpDownDeadQueueName, true, false, false, false, nil) if err != nil { logger.Error(ctx, "declare event dead letter queue failed", "error", err) } - err = channel.QueueBind(deadQueueName, deadRoutingKey, deadExchangeName, false, nil) + err = channel.QueueBind(constants.EventUpDownDeadQueueName, constants.EventUpDownDeadRoutingKey, constants.EventDeadExchangeName, false, nil) if err != nil { logger.Error(ctx, "bind event dead letter queue with routing key and exchange failed", "error", err) } - err = channel.ExchangeDeclare(exchangeName, "topic", true, false, false, false, nil) + err = channel.ExchangeDeclare(constants.EventExchangeName, "topic", true, false, false, false, nil) if err != nil { logger.Error(ctx, "declare event exchange failed", "error", err) } @@ -58,15 +50,15 @@ func initEventAlarmChannel(ctx context.Context) (*amqp.Channel, error) { args := amqp.Table{ // messages that accumulate to the maximum number will be automatically transferred to the dead letter queue "x-max-length": int32(50), - "x-dead-letter-exchange": deadExchangeName, - "x-dead-letter-routing-key": deadRoutingKey, + "x-dead-letter-exchange": constants.EventDeadExchangeName, + "x-dead-letter-routing-key": constants.EventUpDownDeadRoutingKey, } - _, err = channel.QueueDeclare(queueName, true, false, false, false, args) + _, err = channel.QueueDeclare(constants.EventUpDownQueueName, true, false, false, false, args) if err != nil { logger.Error(ctx, "declare event queue failed", "error", err) } - err = channel.QueueBind(queueName, routingKey, exchangeName, false, nil) + err = channel.QueueBind(constants.EventUpDownQueueName, constants.EventUpDownRoutingKey, constants.EventExchangeName, false, nil) if err != nil { logger.Error(ctx, "bind event queue with routing key and exchange failed:", "error", err) } @@ -77,9 +69,9 @@ func initEventAlarmChannel(ctx context.Context) (*amqp.Channel, error) { return channel, nil } -// PushEventToRabbitMQ define func to push event alarm message to rabbitMQ -func PushEventToRabbitMQ(ctx context.Context, msgChan chan []byte) { - channel, err := initEventAlarmChannel(ctx) +// PushUpDownLimitEventToRabbitMQ define func to push up and down limit event message to rabbitMQ +func PushUpDownLimitEventToRabbitMQ(ctx context.Context, msgChan chan []byte) { + channel, err := initUpDownLimitEventChannel(ctx) if err != nil { logger.Error(ctx, "initializing rabbitMQ channel failed", "error", err) return @@ -119,10 +111,10 @@ func PushEventToRabbitMQ(ctx context.Context, msgChan chan []byte) { // send event alarm message to rabbitMQ queue pubCtx, cancel := context.WithTimeout(ctx, 5*time.Second) err = channel.PublishWithContext(pubCtx, - exchangeName, // exchange - routingKey, // routing key - false, // mandatory - false, // immediate + constants.EventExchangeName, // exchange + constants.EventUpDownRoutingKey, // routing key + false, // mandatory + false, // immediate amqp.Publishing{ ContentType: "text/plain", Body: msg, diff --git a/real-time-data/compute_analyzer.go b/real-time-data/compute_analyzer.go index c1ae86a..e1c5166 100644 --- a/real-time-data/compute_analyzer.go +++ b/real-time-data/compute_analyzer.go @@ -131,6 +131,7 @@ func analyzeTEDataLogic(ctx context.Context, conf *ComputeConfig, thresholds teE opts := []event.EventOption{ event.WithConditionValue(triggerValues, conf.Cause), event.WithTEAnalysisResult(firstBreachType), + event.WithCategory(constants.EventUpDownRoutingKey), // TODO 生成 operations并考虑如何放入 event 中 // event.WithOperations(nil) } From 898beaeec482966c33190df71c099bce509333da Mon Sep 17 00:00:00 2001 From: douxu Date: Mon, 2 Mar 2026 17:00:09 +0800 Subject: [PATCH 19/43] optimize struct of rabbitmq event --- common/event_action_errors.go | 10 +++ constants/error.go => common/uuid_errors.go | 4 +- constants/event.go | 13 +++- handler/attr_delete.go | 4 +- handler/attr_load.go | 4 +- handler/attr_update.go | 4 +- handler/diagram_node_link.go | 5 +- handler/measurement_load.go | 4 +- handler/mesurement_link.go | 5 +- handler/real_time_data_subscription.go | 3 +- model/measurement_protol_model.go | 11 +-- {real-time-data => mq}/event/event.go | 0 .../event/event_handlers.go | 67 +++++++------------ {real-time-data => mq}/event/event_options.go | 0 {real-time-data => mq}/event/gen_event.go | 0 mq/publish_up_down_limit_event.go | 43 ++++++++---- network/circuit_diagram_update_request.go | 13 ++-- real-time-data/compute_analyzer.go | 20 ++++-- 18 files changed, 122 insertions(+), 88 deletions(-) create mode 100644 common/event_action_errors.go rename constants/error.go => common/uuid_errors.go (97%) rename {real-time-data => mq}/event/event.go (100%) rename {real-time-data => mq}/event/event_handlers.go (64%) rename {real-time-data => mq}/event/event_options.go (100%) rename {real-time-data => mq}/event/gen_event.go (100%) diff --git a/common/event_action_errors.go b/common/event_action_errors.go new file mode 100644 index 0000000..3df4a39 --- /dev/null +++ b/common/event_action_errors.go @@ -0,0 +1,10 @@ +// Package common define common error variables +package common + +import "errors" + +// ErrUnknowEventActionCommand define error of unknown event action command +var ErrUnknowEventActionCommand = errors.New("unknown action command") + +// ErrExecEventActionFailed define error of execute event action failed +var ErrExecEventActionFailed = errors.New("exec event action func failed") diff --git a/constants/error.go b/common/uuid_errors.go similarity index 97% rename from constants/error.go rename to common/uuid_errors.go index da0b91e..bca1b32 100644 --- a/constants/error.go +++ b/common/uuid_errors.go @@ -1,5 +1,5 @@ -// Package constants define constant variable -package constants +// Package common define common error variables +package common import "errors" diff --git a/constants/event.go b/constants/event.go index 435cb2b..e4ea62e 100644 --- a/constants/event.go +++ b/constants/event.go @@ -73,11 +73,20 @@ const ( const ( // EventUpDownRoutingKey define routing key for up or down limit event alarm message - EventUpDownRoutingKey = "event-up-down-routing-key" + EventUpDownRoutingKey = "event.#" // EventUpDownDeadRoutingKey define dead letter routing key for up or down limit event alarm message - EventUpDownDeadRoutingKey = "event-up-down-dead-letter-routing-key" + EventUpDownDeadRoutingKey = "event.#" // EventUpDownQueueName define queue name for up or down limit event alarm message EventUpDownQueueName = "event-up-down-queue" // EventUpDownDeadQueueName define dead letter queue name for event alarm message EventUpDownDeadQueueName = "event-dead-letter-queue" ) + +const ( + // EventGeneralUpDownLimitCategroy define category for general up and down limit event + EventGeneralUpDownLimitCategroy = "event.general.updown.limit" + // EventWarnUpDownLimitCategroy define category for warn up and down limit event + EventWarnUpDownLimitCategroy = "event.warn.updown.limit" + // EventCriticalUpDownLimitCategroy define category for critical up and down limit event + EventCriticalUpDownLimitCategroy = "event.critical.updown.limit" +) diff --git a/handler/attr_delete.go b/handler/attr_delete.go index b78e46c..fedb49a 100644 --- a/handler/attr_delete.go +++ b/handler/attr_delete.go @@ -3,7 +3,7 @@ package handler import ( "net/http" - "modelRT/constants" + "modelRT/common" "modelRT/diagram" "modelRT/logger" "modelRT/network" @@ -16,7 +16,7 @@ func AttrDeleteHandler(c *gin.Context) { var request network.AttrDeleteRequest clientToken := c.GetString("client_token") if clientToken == "" { - err := constants.ErrGetClientToken + err := common.ErrGetClientToken logger.Error(c, "failed to get client token from context", "error", err) c.JSON(http.StatusOK, network.FailureResponse{ diff --git a/handler/attr_load.go b/handler/attr_load.go index 3a50be6..0c07744 100644 --- a/handler/attr_load.go +++ b/handler/attr_load.go @@ -3,7 +3,7 @@ package handler import ( "net/http" - "modelRT/constants" + "modelRT/common" "modelRT/database" "modelRT/logger" "modelRT/network" @@ -17,7 +17,7 @@ func AttrGetHandler(c *gin.Context) { clientToken := c.GetString("client_token") if clientToken == "" { - err := constants.ErrGetClientToken + err := common.ErrGetClientToken logger.Error(c, "failed to get client token from context", "error", err) c.JSON(http.StatusOK, network.FailureResponse{ diff --git a/handler/attr_update.go b/handler/attr_update.go index 589164e..8a57599 100644 --- a/handler/attr_update.go +++ b/handler/attr_update.go @@ -3,7 +3,7 @@ package handler import ( "net/http" - "modelRT/constants" + "modelRT/common" "modelRT/diagram" "modelRT/logger" "modelRT/network" @@ -17,7 +17,7 @@ func AttrSetHandler(c *gin.Context) { clientToken := c.GetString("client_token") if clientToken == "" { - err := constants.ErrGetClientToken + err := common.ErrGetClientToken logger.Error(c, "failed to get client token from context", "error", err) c.JSON(http.StatusOK, network.FailureResponse{ diff --git a/handler/diagram_node_link.go b/handler/diagram_node_link.go index 80850e7..8f484a5 100644 --- a/handler/diagram_node_link.go +++ b/handler/diagram_node_link.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" + "modelRT/common" "modelRT/constants" "modelRT/database" "modelRT/diagram" @@ -43,7 +44,7 @@ func DiagramNodeLinkHandler(c *gin.Context) { var request network.DiagramNodeLinkRequest clientToken := c.GetString("client_token") if clientToken == "" { - err := constants.ErrGetClientToken + err := common.ErrGetClientToken logger.Error(c, "failed to get client token from context", "error", err) c.JSON(http.StatusOK, network.FailureResponse{ Code: http.StatusBadRequest, @@ -167,7 +168,7 @@ func processLinkSetData(ctx context.Context, action string, level int, prevLinkS err2 = prevLinkSet.SREM(prevMember) } default: - err := constants.ErrUnsupportedLinkAction + err := common.ErrUnsupportedLinkAction logger.Error(ctx, "unsupport diagram node link process action", "action", action, "error", err) return err } diff --git a/handler/measurement_load.go b/handler/measurement_load.go index 065b9f5..26dbbed 100644 --- a/handler/measurement_load.go +++ b/handler/measurement_load.go @@ -4,7 +4,7 @@ package handler import ( "net/http" - "modelRT/constants" + "modelRT/common" "modelRT/database" "modelRT/diagram" "modelRT/logger" @@ -19,7 +19,7 @@ func MeasurementGetHandler(c *gin.Context) { clientToken := c.GetString("client_token") if clientToken == "" { - err := constants.ErrGetClientToken + err := common.ErrGetClientToken logger.Error(c, "failed to get client token from context", "error", err) c.JSON(http.StatusOK, network.FailureResponse{ diff --git a/handler/mesurement_link.go b/handler/mesurement_link.go index b45e8bb..1a878f6 100644 --- a/handler/mesurement_link.go +++ b/handler/mesurement_link.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" + "modelRT/common" "modelRT/constants" "modelRT/database" "modelRT/diagram" @@ -20,7 +21,7 @@ func MeasurementLinkHandler(c *gin.Context) { var request network.MeasurementLinkRequest clientToken := c.GetString("client_token") if clientToken == "" { - err := constants.ErrGetClientToken + err := common.ErrGetClientToken logger.Error(c, "failed to get client token from context", "error", err) c.JSON(http.StatusOK, network.FailureResponse{ Code: http.StatusBadRequest, @@ -93,7 +94,7 @@ func MeasurementLinkHandler(c *gin.Context) { logger.Error(c, "del measurement link process operation failed", "measurement_id", measurementID, "action", action, "error", err) } default: - err = constants.ErrUnsupportedLinkAction + err = common.ErrUnsupportedLinkAction logger.Error(c, "unsupport measurement link process action", "measurement_id", measurementID, "action", action, "error", err) } diff --git a/handler/real_time_data_subscription.go b/handler/real_time_data_subscription.go index 3f171cf..340f48b 100644 --- a/handler/real_time_data_subscription.go +++ b/handler/real_time_data_subscription.go @@ -7,6 +7,7 @@ import ( "maps" "sync" + "modelRT/common" "modelRT/constants" "modelRT/database" "modelRT/logger" @@ -177,7 +178,7 @@ func RealTimeSubHandler(c *gin.Context) { }) return default: - err := fmt.Errorf("%w: request action is %s", constants.ErrUnsupportedSubAction, request.Action) + err := fmt.Errorf("%w: request action is %s", common.ErrUnsupportedSubAction, request.Action) logger.Error(c, "unsupported action of real time data subscription request", "error", err) requestTargetsCount := processRealTimeRequestCount(request.Measurements) results := processRealTimeRequestTargets(request.Measurements, requestTargetsCount, constants.CodeUnsupportSubOperation, err) diff --git a/model/measurement_protol_model.go b/model/measurement_protol_model.go index 9a5446b..7fdce12 100644 --- a/model/measurement_protol_model.go +++ b/model/measurement_protol_model.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "modelRT/common" "modelRT/constants" ) @@ -61,7 +62,7 @@ func generateChannelName(prefix string, number int, suffix string) (string, erro switch prefix { case constants.ChannelPrefixTelemetry: if number > 10 { - return "", constants.ErrExceedsLimitType + return "", common.ErrExceedsLimitType } var builder strings.Builder numberStr := strconv.Itoa(number) @@ -86,7 +87,7 @@ func generateChannelName(prefix string, number int, suffix string) (string, erro channelName := builder.String() return channelName, nil default: - return "", constants.ErrUnsupportedChannelPrefixType + return "", common.ErrUnsupportedChannelPrefixType } } @@ -164,14 +165,14 @@ func (m MeasurementDataSource) GetIOAddress() (IOAddress, error) { if addr, ok := m.IOAddress.(CL3611Address); ok { return addr, nil } - return nil, constants.ErrInvalidAddressType + return nil, common.ErrInvalidAddressType case constants.DataSourceTypePower104: if addr, ok := m.IOAddress.(Power104Address); ok { return addr, nil } - return nil, constants.ErrInvalidAddressType + return nil, common.ErrInvalidAddressType default: - return nil, constants.ErrUnknownDataType + return nil, common.ErrUnknownDataType } } diff --git a/real-time-data/event/event.go b/mq/event/event.go similarity index 100% rename from real-time-data/event/event.go rename to mq/event/event.go diff --git a/real-time-data/event/event_handlers.go b/mq/event/event_handlers.go similarity index 64% rename from real-time-data/event/event_handlers.go rename to mq/event/event_handlers.go index 841b06e..e37e195 100644 --- a/real-time-data/event/event_handlers.go +++ b/mq/event/event_handlers.go @@ -3,13 +3,12 @@ package event import ( "context" - "encoding/json" + "modelRT/common" "modelRT/logger" - "modelRT/mq" ) -type actionHandler func(ctx context.Context, content string, ops ...EventOption) error +type actionHandler func(ctx context.Context, content string, ops ...EventOption) (*EventRecord, error) // actionDispatchMap define variable to store all action handler into map var actionDispatchMap = map[string]actionHandler{ @@ -21,79 +20,63 @@ var actionDispatchMap = map[string]actionHandler{ } // TriggerEventAction define func to trigger event by action in compute config -func TriggerEventAction(ctx context.Context, command string, eventName string, ops ...EventOption) { +func TriggerEventAction(ctx context.Context, command string, eventName string, ops ...EventOption) (*EventRecord, error) { handler, exists := actionDispatchMap[command] if !exists { logger.Error(ctx, "unknown action command", "command", command) - return + return nil, common.ErrUnknowEventActionCommand } - err := handler(ctx, eventName, ops...) + + eventRecord, err := handler(ctx, eventName, ops...) if err != nil { - logger.Error(ctx, "action handler failed", "command", command, "event_name", eventName, "error", err) - return + logger.Error(ctx, "action event handler failed", "error", err) + return nil, common.ErrExecEventActionFailed } - logger.Info(ctx, "action handler success", "command", command, "event_name", eventName) + return eventRecord, nil } -func handleInfoAction(ctx context.Context, eventName string, ops ...EventOption) error { +func handleInfoAction(ctx context.Context, eventName string, ops ...EventOption) (*EventRecord, error) { + logger.Info(ctx, "trigger info event", "event_name", eventName) eventRecord, err := NewGeneralPlatformSoftRecord(eventName, ops...) if err != nil { logger.Error(ctx, "generate info event record failed", "error", err) - return err + return nil, err } - recordBytes, err := json.Marshal(eventRecord) - if err != nil { - logger.Error(ctx, "marshal event record failed", "event_uuid", eventRecord.EventUUID, "error", err) - return err - } - mq.MsgChan <- recordBytes - logger.Info(ctx, "trigger info event", "event_name", eventName) - return nil + return eventRecord, nil } -func handleWarningAction(ctx context.Context, eventName string, ops ...EventOption) error { +func handleWarningAction(ctx context.Context, eventName string, ops ...EventOption) (*EventRecord, error) { + logger.Info(ctx, "trigger warning event", "event_name", eventName) eventRecord, err := NewWarnPlatformSoftRecord(eventName, ops...) if err != nil { logger.Error(ctx, "generate warning event record failed", "error", err) - return err + return nil, err } - recordBytes, err := json.Marshal(eventRecord) - if err != nil { - logger.Error(ctx, "marshal event record failed", "event_uuid", eventRecord.EventUUID, "error", err) - return err - } - mq.MsgChan <- recordBytes - logger.Info(ctx, "trigger warning event", "event_name", eventName) - return nil + return eventRecord, nil } -func handleErrorAction(ctx context.Context, eventName string, ops ...EventOption) error { +func handleErrorAction(ctx context.Context, eventName string, ops ...EventOption) (*EventRecord, error) { + logger.Info(ctx, "trigger error event", "event_name", eventName) eventRecord, err := NewCriticalPlatformSoftRecord(eventName, ops...) if err != nil { logger.Error(ctx, "generate error event record failed", "error", err) - return err + return nil, err } - recordBytes, err := json.Marshal(eventRecord) - if err != nil { - logger.Error(ctx, "marshal event record failed", "event_uuid", eventRecord.EventUUID, "error", err) - return err - } - mq.MsgChan <- recordBytes - return nil + return eventRecord, nil } -func handleCriticalAction(ctx context.Context, content string, ops ...EventOption) error { +func handleCriticalAction(ctx context.Context, content string, ops ...EventOption) (*EventRecord, error) { // 实际执行发送警告、记录日志等操作 actionParams := content // ... logic to send critical level event using actionParams ... logger.Warn(ctx, "trigger critical event", "message", actionParams) - return nil + return nil, nil } -func handleExceptionAction(ctx context.Context, content string, ops ...EventOption) error { +func handleExceptionAction(ctx context.Context, content string, ops ...EventOption) (*EventRecord, error) { // 实际执行发送警告、记录日志等操作 actionParams := content // ... logic to send except level event using actionParams ... logger.Warn(ctx, "trigger except event", "message", actionParams) - return nil + return nil, nil } diff --git a/real-time-data/event/event_options.go b/mq/event/event_options.go similarity index 100% rename from real-time-data/event/event_options.go rename to mq/event/event_options.go diff --git a/real-time-data/event/gen_event.go b/mq/event/gen_event.go similarity index 100% rename from real-time-data/event/gen_event.go rename to mq/event/gen_event.go diff --git a/mq/publish_up_down_limit_event.go b/mq/publish_up_down_limit_event.go index e1f0bd1..9d4bbbf 100644 --- a/mq/publish_up_down_limit_event.go +++ b/mq/publish_up_down_limit_event.go @@ -3,19 +3,21 @@ package mq import ( "context" + "encoding/json" "time" "modelRT/constants" "modelRT/logger" + "modelRT/mq/event" amqp "github.com/rabbitmq/amqp091-go" ) // MsgChan define variable of channel to store messages that need to be sent to rabbitMQ -var MsgChan chan []byte +var MsgChan chan *event.EventRecord func init() { - MsgChan = make(chan []byte, 10000) + MsgChan = make(chan *event.EventRecord, 10000) } func initUpDownLimitEventChannel(ctx context.Context) (*amqp.Channel, error) { @@ -25,30 +27,34 @@ func initUpDownLimitEventChannel(ctx context.Context) (*amqp.Channel, error) { channel, err = GetConn().Channel() if err != nil { logger.Error(ctx, "open rabbitMQ server channel failed", "error", err) + return nil, err } err = channel.ExchangeDeclare(constants.EventDeadExchangeName, "topic", true, false, false, false, nil) if err != nil { logger.Error(ctx, "declare event dead letter exchange failed", "error", err) + return nil, err } _, err = channel.QueueDeclare(constants.EventUpDownDeadQueueName, true, false, false, false, nil) if err != nil { logger.Error(ctx, "declare event dead letter queue failed", "error", err) + return nil, err } - err = channel.QueueBind(constants.EventUpDownDeadQueueName, constants.EventUpDownDeadRoutingKey, constants.EventDeadExchangeName, false, nil) + err = channel.QueueBind(constants.EventUpDownDeadQueueName, "#", constants.EventDeadExchangeName, false, nil) if err != nil { logger.Error(ctx, "bind event dead letter queue with routing key and exchange failed", "error", err) + return nil, err } err = channel.ExchangeDeclare(constants.EventExchangeName, "topic", true, false, false, false, nil) if err != nil { logger.Error(ctx, "declare event exchange failed", "error", err) + return nil, err } args := amqp.Table{ - // messages that accumulate to the maximum number will be automatically transferred to the dead letter queue "x-max-length": int32(50), "x-dead-letter-exchange": constants.EventDeadExchangeName, "x-dead-letter-routing-key": constants.EventUpDownDeadRoutingKey, @@ -56,21 +62,24 @@ func initUpDownLimitEventChannel(ctx context.Context) (*amqp.Channel, error) { _, err = channel.QueueDeclare(constants.EventUpDownQueueName, true, false, false, false, args) if err != nil { logger.Error(ctx, "declare event queue failed", "error", err) + return nil, err } err = channel.QueueBind(constants.EventUpDownQueueName, constants.EventUpDownRoutingKey, constants.EventExchangeName, false, nil) if err != nil { - logger.Error(ctx, "bind event queue with routing key and exchange failed:", "error", err) + logger.Error(ctx, "bind event queue with routing key and exchange failed", "error", err) + return nil, err } if err := channel.Confirm(false); err != nil { logger.Error(ctx, "channel could not be put into confirm mode", "error", err) + return nil, err } return channel, nil } // PushUpDownLimitEventToRabbitMQ define func to push up and down limit event message to rabbitMQ -func PushUpDownLimitEventToRabbitMQ(ctx context.Context, msgChan chan []byte) { +func PushUpDownLimitEventToRabbitMQ(ctx context.Context, msgChan chan *event.EventRecord) { channel, err := initUpDownLimitEventChannel(ctx) if err != nil { logger.Error(ctx, "initializing rabbitMQ channel failed", "error", err) @@ -101,28 +110,36 @@ func PushUpDownLimitEventToRabbitMQ(ctx context.Context, msgChan chan []byte) { logger.Info(ctx, "push event alarm message to rabbitMQ stopped by context cancel") channel.Close() return - case msg, ok := <-msgChan: + case eventRecord, ok := <-msgChan: if !ok { logger.Info(ctx, "push event alarm message to rabbitMQ stopped by msgChan closed, exiting push loop") channel.Close() return } + // TODO 将消息的序列化移动到发送之前，以便使用eventRecord的category来作为routing key + recordBytes, err := json.Marshal(eventRecord) + if err != nil { + logger.Error(ctx, "marshal event record failed", "event_uuid", eventRecord.EventUUID, "error", err) + continue + } + // send event alarm message to rabbitMQ queue + routingKey := eventRecord.Category pubCtx, cancel := context.WithTimeout(ctx, 5*time.Second) err = channel.PublishWithContext(pubCtx, - constants.EventExchangeName, // exchange - constants.EventUpDownRoutingKey, // routing key - false, // mandatory - false, // immediate + constants.EventExchangeName, // exchange + routingKey, // routing key + false, // mandatory + false, // immediate amqp.Publishing{ ContentType: "text/plain", - Body: msg, + Body: recordBytes, }) cancel() if err != nil { - logger.Error(ctx, "publish message to rabbitMQ queue failed", "message", msg, "error", err) + logger.Error(ctx, "publish message to rabbitMQ queue failed", "message", recordBytes, "error", err) } } } diff --git a/network/circuit_diagram_update_request.go b/network/circuit_diagram_update_request.go index 06966cd..86b1a76 100644 --- a/network/circuit_diagram_update_request.go +++ b/network/circuit_diagram_update_request.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "modelRT/common" "modelRT/common/errcode" "modelRT/constants" "modelRT/orm" @@ -64,10 +65,10 @@ func ParseUUID(info TopologicChangeInfo) (TopologicUUIDChangeInfos, error) { switch info.ChangeType { case constants.UUIDFromChangeType: if info.NewUUIDFrom == info.OldUUIDFrom { - return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", constants.ErrUUIDFromCheckT1) + return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", common.ErrUUIDFromCheckT1) } if info.NewUUIDTo != info.OldUUIDTo { - return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", constants.ErrUUIDToCheckT1) + return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", common.ErrUUIDToCheckT1) } oldUUIDFrom, err := uuid.FromString(info.OldUUIDFrom) @@ -90,10 +91,10 @@ func ParseUUID(info TopologicChangeInfo) (TopologicUUIDChangeInfos, error) { UUIDChangeInfo.NewUUIDTo = OldUUIDTo case constants.UUIDToChangeType: if info.NewUUIDFrom != info.OldUUIDFrom { - return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", constants.ErrUUIDFromCheckT2) + return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", common.ErrUUIDFromCheckT2) } if info.NewUUIDTo == info.OldUUIDTo { - return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", constants.ErrUUIDToCheckT2) + return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", common.ErrUUIDToCheckT2) } oldUUIDFrom, err := uuid.FromString(info.OldUUIDFrom) @@ -116,10 +117,10 @@ func ParseUUID(info TopologicChangeInfo) (TopologicUUIDChangeInfos, error) { UUIDChangeInfo.NewUUIDTo = newUUIDTo case constants.UUIDAddChangeType: if info.OldUUIDFrom != "" { - return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", constants.ErrUUIDFromCheckT3) + return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", common.ErrUUIDFromCheckT3) } if info.OldUUIDTo != "" { - return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", constants.ErrUUIDToCheckT3) + return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", common.ErrUUIDToCheckT3) } newUUIDFrom, err := uuid.FromString(info.NewUUIDFrom) diff --git a/real-time-data/compute_analyzer.go b/real-time-data/compute_analyzer.go index e1c5166..d65a343 100644 --- a/real-time-data/compute_analyzer.go +++ b/real-time-data/compute_analyzer.go @@ -9,7 +9,8 @@ import ( "modelRT/constants" "modelRT/logger" - "modelRT/real-time-data/event" + "modelRT/mq" + "modelRT/mq/event" ) // RealTimeAnalyzer define interface general methods for real-time data analysis and event triggering @@ -131,7 +132,7 @@ func analyzeTEDataLogic(ctx context.Context, conf *ComputeConfig, thresholds teE opts := []event.EventOption{ event.WithConditionValue(triggerValues, conf.Cause), event.WithTEAnalysisResult(firstBreachType), - event.WithCategory(constants.EventUpDownRoutingKey), + event.WithCategory(constants.EventWarnUpDownLimitCategroy), // TODO 生成 operations并考虑如何放入 event 中 // event.WithOperations(nil) } @@ -149,8 +150,12 @@ func analyzeTEDataLogic(ctx context.Context, conf *ComputeConfig, thresholds teE // trigger Action command, mainBody := genTEEventCommandAndMainBody(ctx, conf.Action) eventName := fmt.Sprintf("telemetry_%s_%s_Breach_Event", mainBody, breachType) - event.TriggerEventAction(ctx, command, eventName, trigger.eventOpts...) - + eventRecord, err := event.TriggerEventAction(ctx, command, eventName, trigger.eventOpts...) + if err != nil { + logger.Error(ctx, "trigger event action failed", "error", err) + return + } + mq.MsgChan <- eventRecord } } @@ -320,7 +325,12 @@ func analyzeTIDataLogic(ctx context.Context, conf *ComputeConfig, thresholds tiE logger.Error(ctx, "generate telemetry evnet command or content failed", "action", conf.Action, "command", command, "main_body", mainBody) return } - event.TriggerEventAction(ctx, command, mainBody) + eventRecord, err := event.TriggerEventAction(ctx, command, mainBody) + if err != nil { + logger.Error(ctx, "trigger event action failed", "error", err) + return + } + mq.MsgChan <- eventRecord return } } From a94abdb47986aa0fdefde461f84ccb3ee04b67e7 Mon Sep 17 00:00:00 2001 From: douxu Date: Thu, 5 Mar 2026 17:15:51 +0800 Subject: [PATCH 20/43] initialize the asynchronous task system's initial structure --- task/types.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 task/types.go diff --git a/task/types.go b/task/types.go new file mode 100644 index 0000000..8ea5c73 --- /dev/null +++ b/task/types.go @@ -0,0 +1,55 @@ +package task + +import ( + "time" +) + +type TaskStatus string + +const ( + StatusPending TaskStatus = "PENDING" + StatusRunning TaskStatus = "RUNNING" + StatusCompleted TaskStatus = "COMPLETED" + StatusFailed TaskStatus = "FAILED" +) + +// TaskType 定义异步任务的具体业务类型 +type TaskType string + +const ( + TypeTopologyAnalysis TaskType = "TOPOLOGY_ANALYSIS" + TypeEventAnalysis TaskType = "EVENT_ANALYSIS" + TypeBatchImport TaskType = "BATCH_IMPORT" +) + +type Task struct { + ID string `bson:"_id" json:"id"` + Type TaskType `bson:"type" json:"type"` + Status TaskStatus `bson:"status" json:"status"` + Priority int `bson:"priority" json:"priority"` + + Params map[string]interface{} `bson:"params" json:"params"` + Result map[string]interface{} `bson:"result,omitempty" json:"result"` + ErrorMsg string `bson:"error_msg,omitempty" json:"error_msg"` + + CreatedAt time.Time `bson:"created_at" json:"created_at"` + StartedAt time.Time `bson:"started_at,omitempty" json:"started_at"` + CompletedAt time.Time `bson:"completed_at,omitempty" json:"completed_at"` +} + +type TopologyParams struct { + CheckIsland bool `json:"check_island"` + CheckShort bool `json:"check_short"` + BaseModelIDs []string `json:"base_model_ids"` +} + +type EventAnalysisParams struct { + MotorID string `json:"motor_id"` + TriggerID string `json:"trigger_id"` + DurationMS int `json:"duration_ms"` +} + +type BatchImportParams struct { + FileName string `json:"file_name"` + FilePath string `json:"file_path"` +} From 6e0d2186d81c10105314a8a9f57a254834526c8e Mon Sep 17 00:00:00 2001 From: douxu Date: Thu, 12 Mar 2026 16:37:06 +0800 Subject: [PATCH 21/43] optimize code of async task system --- orm/async_task.go | 115 +++++++++++++++++++++++++++++++++++++++ orm/async_task_result.go | 70 ++++++++++++++++++++++++ task/queue_message.go | 77 ++++++++++++++++++++++++++ 3 files changed, 262 insertions(+) create mode 100644 orm/async_task.go create mode 100644 orm/async_task_result.go create mode 100644 task/queue_message.go diff --git a/orm/async_task.go b/orm/async_task.go new file mode 100644 index 0000000..52a03c6 --- /dev/null +++ b/orm/async_task.go @@ -0,0 +1,115 @@ +// Package orm define database data struct +package orm + +import ( + "github.com/gofrs/uuid" +) + +// AsyncTaskType defines the type of asynchronous task +type AsyncTaskType string + +const ( + // AsyncTaskTypeTopologyAnalysis represents topology analysis task + AsyncTaskTypeTopologyAnalysis AsyncTaskType = "TOPOLOGY_ANALYSIS" + // AsyncTaskTypePerformanceAnalysis represents performance analysis task + AsyncTaskTypePerformanceAnalysis AsyncTaskType = "PERFORMANCE_ANALYSIS" + // AsyncTaskTypeEventAnalysis represents event analysis task + AsyncTaskTypeEventAnalysis AsyncTaskType = "EVENT_ANALYSIS" + // AsyncTaskTypeBatchImport represents batch import task + AsyncTaskTypeBatchImport AsyncTaskType = "BATCH_IMPORT" +) + +// AsyncTaskStatus defines the status of asynchronous task +type AsyncTaskStatus string + +const ( + // AsyncTaskStatusSubmitted represents task has been submitted to queue + AsyncTaskStatusSubmitted AsyncTaskStatus = "SUBMITTED" + // AsyncTaskStatusRunning represents task is currently executing + AsyncTaskStatusRunning AsyncTaskStatus = "RUNNING" + // AsyncTaskStatusCompleted represents task completed successfully + AsyncTaskStatusCompleted AsyncTaskStatus = "COMPLETED" + // AsyncTaskStatusFailed represents task failed with error + AsyncTaskStatusFailed AsyncTaskStatus = "FAILED" +) + +// AsyncTask defines the core task entity stored in database for task lifecycle tracking +type AsyncTask struct { + TaskID uuid.UUID `gorm:"column:task_id;primaryKey;type:uuid;default:gen_random_uuid()"` + TaskType AsyncTaskType `gorm:"column:task_type;type:varchar(50);not null"` + Status AsyncTaskStatus `gorm:"column:status;type:varchar(20);not null;index"` + Params JSONMap `gorm:"column:params;type:jsonb"` + CreatedAt int64 `gorm:"column:created_at;not null;index"` + FinishedAt *int64 `gorm:"column:finished_at;index"` + Progress *int `gorm:"column:progress"` // 0-100, nullable +} + +// TableName returns the table name for AsyncTask model +func (a *AsyncTask) TableName() string { + return "async_task" +} + +// SetSubmitted marks the task as submitted +func (a *AsyncTask) SetSubmitted() { + a.Status = AsyncTaskStatusSubmitted +} + +// SetRunning marks the task as running +func (a *AsyncTask) SetRunning() { + a.Status = AsyncTaskStatusRunning +} + +// SetCompleted marks the task as completed with finished timestamp +func (a *AsyncTask) SetCompleted(timestamp int64) { + a.Status = AsyncTaskStatusCompleted + a.FinishedAt = ×tamp + a.setProgress(100) +} + +// SetFailed marks the task as failed with finished timestamp +func (a *AsyncTask) SetFailed(timestamp int64) { + a.Status = AsyncTaskStatusFailed + a.FinishedAt = ×tamp +} + +// setProgress updates the task progress (0-100) +func (a *AsyncTask) setProgress(value int) { + if value < 0 { + value = 0 + } + if value > 100 { + value = 100 + } + a.Progress = &value +} + +// UpdateProgress updates the task progress with validation +func (a *AsyncTask) UpdateProgress(value int) { + a.setProgress(value) +} + +// IsCompleted checks if the task is completed +func (a *AsyncTask) IsCompleted() bool { + return a.Status == AsyncTaskStatusCompleted +} + +// IsRunning checks if the task is running +func (a *AsyncTask) IsRunning() bool { + return a.Status == AsyncTaskStatusRunning +} + +// IsFailed checks if the task failed +func (a *AsyncTask) IsFailed() bool { + return a.Status == AsyncTaskStatusFailed +} + +// IsValidTaskType checks if the task type is valid +func IsValidAsyncTaskType(taskType string) bool { + switch AsyncTaskType(taskType) { + case AsyncTaskTypeTopologyAnalysis, AsyncTaskTypePerformanceAnalysis, + AsyncTaskTypeEventAnalysis, AsyncTaskTypeBatchImport: + return true + default: + return false + } +} diff --git a/orm/async_task_result.go b/orm/async_task_result.go new file mode 100644 index 0000000..a3c6213 --- /dev/null +++ b/orm/async_task_result.go @@ -0,0 +1,70 @@ +// Package orm define database data struct +package orm + +import ( + "github.com/gofrs/uuid" +) + +// AsyncTaskResult stores computation results, separate from AsyncTask model for flexibility +type AsyncTaskResult struct { + TaskID uuid.UUID `gorm:"column:task_id;primaryKey;type:uuid"` + Result JSONMap `gorm:"column:result;type:jsonb"` + ErrorCode *int `gorm:"column:error_code"` + ErrorMessage *string `gorm:"column:error_message;type:text"` + ErrorDetail JSONMap `gorm:"column:error_detail;type:jsonb"` +} + +// TableName returns the table name for AsyncTaskResult model +func (a *AsyncTaskResult) TableName() string { + return "async_task_result" +} + +// SetSuccess sets the result for successful task execution +func (a *AsyncTaskResult) SetSuccess(result JSONMap) { + a.Result = result + a.ErrorCode = nil + a.ErrorMessage = nil + a.ErrorDetail = nil +} + +// SetError sets the error information for failed task execution +func (a *AsyncTaskResult) SetError(code int, message string, detail JSONMap) { + a.Result = nil + a.ErrorCode = &code + a.ErrorMessage = &message + a.ErrorDetail = detail +} + +// HasError checks if the task result contains an error +func (a *AsyncTaskResult) HasError() bool { + return a.ErrorCode != nil || a.ErrorMessage != nil +} + +// GetErrorCode returns the error code or 0 if no error +func (a *AsyncTaskResult) GetErrorCode() int { + if a.ErrorCode == nil { + return 0 + } + return *a.ErrorCode +} + +// GetErrorMessage returns the error message or empty string if no error +func (a *AsyncTaskResult) GetErrorMessage() string { + if a.ErrorMessage == nil { + return "" + } + return *a.ErrorMessage +} + +// IsSuccess checks if the task execution was successful +func (a *AsyncTaskResult) IsSuccess() bool { + return !a.HasError() +} + +// Clear clears all result data +func (a *AsyncTaskResult) Clear() { + a.Result = nil + a.ErrorCode = nil + a.ErrorMessage = nil + a.ErrorDetail = nil +} diff --git a/task/queue_message.go b/task/queue_message.go new file mode 100644 index 0000000..0c6512f --- /dev/null +++ b/task/queue_message.go @@ -0,0 +1,77 @@ +package task + +import ( + "github.com/gofrs/uuid" +) + +// DefaultPriority is the default task priority +const DefaultPriority = 5 + +// HighPriority represents high priority tasks +const HighPriority = 10 + +// LowPriority represents low priority tasks +const LowPriority = 1 + +// TaskQueueMessage defines minimal message structure for RabbitMQ/Redis queue dispatch +// This struct is designed to be lightweight for efficient message transport +type TaskQueueMessage struct { + TaskID uuid.UUID `json:"task_id"` + TaskType TaskType `json:"task_type"` + Priority int `json:"priority,omitempty"` // Optional, defaults to DefaultPriority +} + +// NewTaskQueueMessage creates a new TaskQueueMessage with default priority +func NewTaskQueueMessage(taskID uuid.UUID, taskType TaskType) *TaskQueueMessage { + return &TaskQueueMessage{ + TaskID: taskID, + TaskType: taskType, + Priority: DefaultPriority, + } +} + +// NewTaskQueueMessageWithPriority creates a new TaskQueueMessage with specified priority +func NewTaskQueueMessageWithPriority(taskID uuid.UUID, taskType TaskType, priority int) *TaskQueueMessage { + return &TaskQueueMessage{ + TaskID: taskID, + TaskType: taskType, + Priority: priority, + } +} + +// ToJSON converts the TaskQueueMessage to JSON bytes +func (m *TaskQueueMessage) ToJSON() ([]byte, error) { + return []byte{}, nil // Placeholder - actual implementation would use json.Marshal +} + +// Validate checks if the TaskQueueMessage is valid +func (m *TaskQueueMessage) Validate() bool { + // Check if TaskID is valid (not nil UUID) + if m.TaskID == uuid.Nil { + return false + } + + // Check if TaskType is valid + switch m.TaskType { + case TypeTopologyAnalysis, TypeEventAnalysis, TypeBatchImport: + return true + default: + return false + } +} + +// SetPriority sets the priority of the task queue message with validation +func (m *TaskQueueMessage) SetPriority(priority int) { + if priority < LowPriority { + priority = LowPriority + } + if priority > HighPriority { + priority = HighPriority + } + m.Priority = priority +} + +// GetPriority returns the priority of the task queue message +func (m *TaskQueueMessage) GetPriority() int { + return m.Priority +} From adcc8c6c915a4142add8af5ae12ca709f09a94fa Mon Sep 17 00:00:00 2001 From: douxu Date: Fri, 13 Mar 2026 11:45:22 +0800 Subject: [PATCH 22/43] add code of async task system --- database/async_task_operations.go | 321 ++++++++++++++++++++++++++++++ database/postgres_init.go | 11 + network/async_task_request.go | 95 +++++++++ 3 files changed, 427 insertions(+) create mode 100644 database/async_task_operations.go create mode 100644 network/async_task_request.go diff --git a/database/async_task_operations.go b/database/async_task_operations.go new file mode 100644 index 0000000..991e77b --- /dev/null +++ b/database/async_task_operations.go @@ -0,0 +1,321 @@ +// Package database define database operation functions +package database + +import ( + "context" + "time" + + "modelRT/orm" + + "github.com/gofrs/uuid" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// CreateAsyncTask creates a new async task in the database +func CreateAsyncTask(ctx context.Context, tx *gorm.DB, taskType orm.AsyncTaskType, params orm.JSONMap) (*orm.AsyncTask, error) { + taskID, err := uuid.NewV4() + if err != nil { + return nil, err + } + + task := &orm.AsyncTask{ + TaskID: taskID, + TaskType: taskType, + Status: orm.AsyncTaskStatusSubmitted, + Params: params, + CreatedAt: time.Now().Unix(), + } + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx).Create(task) + if result.Error != nil { + return nil, result.Error + } + + return task, nil +} + +// GetAsyncTaskByID retrieves an async task by its ID +func GetAsyncTaskByID(ctx context.Context, tx *gorm.DB, taskID uuid.UUID) (*orm.AsyncTask, error) { + var task orm.AsyncTask + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("task_id = ?", taskID). + Clauses(clause.Locking{Strength: "UPDATE"}). + First(&task) + + if result.Error != nil { + return nil, result.Error + } + + return &task, nil +} + +// GetAsyncTasksByIDs retrieves multiple async tasks by their IDs +func GetAsyncTasksByIDs(ctx context.Context, tx *gorm.DB, taskIDs []uuid.UUID) ([]orm.AsyncTask, error) { + var tasks []orm.AsyncTask + + if len(taskIDs) == 0 { + return tasks, nil + } + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("task_id IN ?", taskIDs). + Clauses(clause.Locking{Strength: "UPDATE"}). + Find(&tasks) + + if result.Error != nil { + return nil, result.Error + } + + return tasks, nil +} + +// UpdateAsyncTaskStatus updates the status of an async task +func UpdateAsyncTaskStatus(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, status orm.AsyncTaskStatus) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("status", status) + + return result.Error +} + +// UpdateAsyncTaskProgress updates the progress of an async task +func UpdateAsyncTaskProgress(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, progress int) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("progress", progress) + + return result.Error +} + +// CompleteAsyncTask marks an async task as completed with timestamp +func CompleteAsyncTask(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, timestamp int64) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "status": orm.AsyncTaskStatusCompleted, + "finished_at": timestamp, + "progress": 100, + }) + + return result.Error +} + +// FailAsyncTask marks an async task as failed with timestamp +func FailAsyncTask(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, timestamp int64) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "status": orm.AsyncTaskStatusFailed, + "finished_at": timestamp, + }) + + return result.Error +} + +// CreateAsyncTaskResult creates a result record for an async task +func CreateAsyncTaskResult(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, result orm.JSONMap) error { + taskResult := &orm.AsyncTaskResult{ + TaskID: taskID, + Result: result, + } + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + resultOp := tx.WithContext(cancelCtx).Create(taskResult) + return resultOp.Error +} + +// UpdateAsyncTaskResultWithError updates a task result with error information +func UpdateAsyncTaskResultWithError(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, code int, message string, detail orm.JSONMap) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + // Update with error information + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTaskResult{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "error_code": code, + "error_message": message, + "error_detail": detail, + "result": nil, + }) + + return result.Error +} + +// UpdateAsyncTaskResultWithSuccess updates a task result with success information +func UpdateAsyncTaskResultWithSuccess(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, result orm.JSONMap) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + // First try to update existing record, if not found create new one + existingResult := tx.WithContext(cancelCtx). + Where("task_id = ?", taskID). + FirstOrCreate(&orm.AsyncTaskResult{TaskID: taskID}) + + if existingResult.Error != nil { + return existingResult.Error + } + + // Update with success information + updateResult := tx.WithContext(cancelCtx). + Model(&orm.AsyncTaskResult{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "result": result, + "error_code": nil, + "error_message": nil, + "error_detail": nil, + }) + + return updateResult.Error +} + +// GetAsyncTaskResult retrieves the result of an async task +func GetAsyncTaskResult(ctx context.Context, tx *gorm.DB, taskID uuid.UUID) (*orm.AsyncTaskResult, error) { + var taskResult orm.AsyncTaskResult + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("task_id = ?", taskID). + First(&taskResult) + + if result.Error != nil { + if result.Error == gorm.ErrRecordNotFound { + return nil, nil + } + return nil, result.Error + } + + return &taskResult, nil +} + +// GetAsyncTaskResults retrieves multiple task results by task IDs +func GetAsyncTaskResults(ctx context.Context, tx *gorm.DB, taskIDs []uuid.UUID) ([]orm.AsyncTaskResult, error) { + var taskResults []orm.AsyncTaskResult + + if len(taskIDs) == 0 { + return taskResults, nil + } + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("task_id IN ?", taskIDs). + Find(&taskResults) + + if result.Error != nil { + return nil, result.Error + } + + return taskResults, nil +} + +// GetPendingTasks retrieves pending tasks (submitted but not yet running/completed) +func GetPendingTasks(ctx context.Context, tx *gorm.DB, limit int) ([]orm.AsyncTask, error) { + var tasks []orm.AsyncTask + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("status = ?", orm.AsyncTaskStatusSubmitted). + Order("created_at ASC"). + Limit(limit). + Find(&tasks) + + if result.Error != nil { + return nil, result.Error + } + + return tasks, nil +} + +// GetTasksByStatus retrieves tasks by status +func GetTasksByStatus(ctx context.Context, tx *gorm.DB, status orm.AsyncTaskStatus, limit int) ([]orm.AsyncTask, error) { + var tasks []orm.AsyncTask + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("status = ?", status). + Order("created_at ASC"). + Limit(limit). + Find(&tasks) + + if result.Error != nil { + return nil, result.Error + } + + return tasks, nil +} + +// DeleteOldTasks deletes tasks older than the specified timestamp +func DeleteOldTasks(ctx context.Context, tx *gorm.DB, olderThan int64) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + // First delete task results + result := tx.WithContext(cancelCtx). + Where("task_id IN (SELECT task_id FROM async_task WHERE created_at < ?)", olderThan). + Delete(&orm.AsyncTaskResult{}) + + if result.Error != nil { + return result.Error + } + + // Then delete tasks + result = tx.WithContext(cancelCtx). + Where("created_at < ?", olderThan). + Delete(&orm.AsyncTask{}) + + return result.Error +} \ No newline at end of file diff --git a/database/postgres_init.go b/database/postgres_init.go index fdba0ee..d2babd4 100644 --- a/database/postgres_init.go +++ b/database/postgres_init.go @@ -5,6 +5,7 @@ import ( "sync" "modelRT/logger" + "modelRT/orm" "gorm.io/driver/postgres" "gorm.io/gorm" @@ -34,5 +35,15 @@ func initPostgresDBClient(PostgresDBURI string) *gorm.DB { if err != nil { panic(err) } + + // Auto migrate async task tables + err = db.AutoMigrate( + &orm.AsyncTask{}, + &orm.AsyncTaskResult{}, + ) + if err != nil { + panic(err) + } + return db } diff --git a/network/async_task_request.go b/network/async_task_request.go new file mode 100644 index 0000000..9534b76 --- /dev/null +++ b/network/async_task_request.go @@ -0,0 +1,95 @@ +// Package network define struct of network operation +package network + +import ( + "time" + + "github.com/gofrs/uuid" +) + +// AsyncTaskCreateRequest defines the request structure for creating an asynchronous task +type AsyncTaskCreateRequest struct { + // required: true + // enum: TOPOLOGY_ANALYSIS, PERFORMANCE_ANALYSIS, EVENT_ANALYSIS, BATCH_IMPORT + TaskType string `json:"task_type" example:"TOPOLOGY_ANALYSIS" description:"异步任务类型"` + // required: true + Params map[string]interface{} `json:"params" swaggertype:"object" description:"任务参数，根据任务类型不同而不同"` +} + +// AsyncTaskCreateResponse defines the response structure for creating an asynchronous task +type AsyncTaskCreateResponse struct { + TaskID uuid.UUID `json:"task_id" example:"123e4567-e89b-12d3-a456-426614174000" description:"任务唯一标识符"` +} + +// AsyncTaskResultQueryRequest defines the request structure for querying task results +type AsyncTaskResultQueryRequest struct { + // required: true + TaskIDs []uuid.UUID `json:"task_ids" swaggertype:"array,string" example:"[\"123e4567-e89b-12d3-a456-426614174000\",\"223e4567-e89b-12d3-a456-426614174001\"]" description:"任务ID列表"` +} + +// AsyncTaskResult defines the structure for a single task result +type AsyncTaskResult struct { + TaskID uuid.UUID `json:"task_id" example:"123e4567-e89b-12d3-a456-426614174000" description:"任务唯一标识符"` + TaskType string `json:"task_type" example:"TOPOLOGY_ANALYSIS" description:"任务类型"` + Status string `json:"status" example:"COMPLETED" description:"任务状态：SUBMITTED, RUNNING, COMPLETED, FAILED"` + Progress *int `json:"progress,omitempty" example:"65" description:"任务进度(0-100)，仅当状态为RUNNING时返回"` + CreatedAt int64 `json:"created_at" example:"1741846200" description:"任务创建时间戳"` + FinishedAt *int64 `json:"finished_at,omitempty" example:"1741846205" description:"任务完成时间戳，仅当状态为COMPLETED或FAILED时返回"` + Result map[string]interface{} `json:"result,omitempty" swaggertype:"object" description:"任务结果，仅当状态为COMPLETED时返回"` + ErrorCode *int `json:"error_code,omitempty" example:"400102" description:"错误码，仅当状态为FAILED时返回"` + ErrorMessage *string `json:"error_message,omitempty" example:"Component UUID not found" description:"错误信息，仅当状态为FAILED时返回"` + ErrorDetail map[string]interface{} `json:"error_detail,omitempty" swaggertype:"object" description:"错误详情，仅当状态为FAILED时返回"` +} + +// AsyncTaskResultQueryResponse defines the response structure for querying task results +type AsyncTaskResultQueryResponse struct { + Total int `json:"total" example:"3" description:"查询的任务总数"` + Tasks []AsyncTaskResult `json:"tasks" description:"任务结果列表"` +} + +// AsyncTaskProgressUpdate defines the structure for task progress update +type AsyncTaskProgressUpdate struct { + TaskID uuid.UUID `json:"task_id" example:"123e4567-e89b-12d3-a456-426614174000" description:"任务唯一标识符"` + Progress int `json:"progress" example:"50" description:"任务进度(0-100)"` +} + +// AsyncTaskStatusUpdate defines the structure for task status update +type AsyncTaskStatusUpdate struct { + TaskID uuid.UUID `json:"task_id" example:"123e4567-e89b-12d3-a456-426614174000" description:"任务唯一标识符"` + Status string `json:"status" example:"RUNNING" description:"任务状态：SUBMITTED, RUNNING, COMPLETED, FAILED"` + Timestamp int64 `json:"timestamp" example:"1741846205" description:"状态更新时间戳"` +} + +// TopologyAnalysisParams defines the parameters for topology analysis task +type TopologyAnalysisParams struct { + StartUUID string `json:"start_uuid" example:"comp-001" description:"起始元件UUID"` + EndUUID string `json:"end_uuid" example:"comp-999" description:"目标元件UUID"` +} + +// PerformanceAnalysisParams defines the parameters for performance analysis task +type PerformanceAnalysisParams struct { + ComponentIDs []string `json:"component_ids" example:"[\"comp-001\",\"comp-002\"]" description:"需要分析的元件ID列表"` + TimeRange struct { + Start time.Time `json:"start" example:"2026-03-01T00:00:00Z" description:"分析开始时间"` + End time.Time `json:"end" example:"2026-03-02T00:00:00Z" description:"分析结束时间"` + } `json:"time_range" description:"分析时间范围"` +} + +// EventAnalysisParams defines the parameters for event analysis task +type EventAnalysisParams struct { + EventType string `json:"event_type" example:"MOTOR_START" description:"事件类型"` + StartTime time.Time `json:"start_time" example:"2026-03-01T00:00:00Z" description:"事件开始时间"` + EndTime time.Time `json:"end_time" example:"2026-03-02T00:00:00Z" description:"事件结束时间"` + Components []string `json:"components,omitempty" example:"[\"comp-001\",\"comp-002\"]" description:"关联的元件列表"` +} + +// BatchImportParams defines the parameters for batch import task +type BatchImportParams struct { + FilePath string `json:"file_path" example:"/data/import/model.csv" description:"导入文件路径"` + FileType string `json:"file_type" example:"CSV" description:"文件类型：CSV, JSON, XML"` + Options struct { + Overwrite bool `json:"overwrite" example:"false" description:"是否覆盖现有数据"` + Validate bool `json:"validate" example:"true" description:"是否进行数据验证"` + NotifyUser bool `json:"notify_user" example:"true" description:"是否通知用户"` + } `json:"options" description:"导入选项"` +} From de5f976c317411d53c523c52c01d0f2c4c2b33c7 Mon Sep 17 00:00:00 2001 From: douxu Date: Tue, 17 Mar 2026 16:08:46 +0800 Subject: [PATCH 23/43] add route of async task system --- .gitignore | 12 + handler/async_task_handler.go | 677 ++++++++++++++++++++++++++++++++++ router/async_task.go | 32 ++ router/router.go | 1 + 4 files changed, 722 insertions(+) create mode 100644 handler/async_task_handler.go create mode 100644 router/async_task.go diff --git a/.gitignore b/.gitignore index 673582e..b338b88 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,15 @@ go.work # Shield config files in the configs folder /configs/**/*.yaml /configs/**/*.pem + +# ai config +.cursor/ +.claude/ +.cursorrules +.copilot/ +.chatgpt/ +.ai_history/ +.vector_cache/ +ai-debug.log +*.patch +*.diff \ No newline at end of file diff --git a/handler/async_task_handler.go b/handler/async_task_handler.go new file mode 100644 index 0000000..9808e33 --- /dev/null +++ b/handler/async_task_handler.go @@ -0,0 +1,677 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "net/http" + "strings" + "time" + + "modelRT/database" + "modelRT/logger" + "modelRT/network" + "modelRT/orm" + _ "modelRT/task" + + "github.com/gin-gonic/gin" + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// AsyncTaskCreateHandler handles creation of asynchronous tasks +// @Summary 创建异步任务 +// @Description 创建新的异步任务并返回任务ID，任务将被提交到队列等待处理 +// @Tags AsyncTask +// @Accept json +// @Produce json +// @Param request body network.AsyncTaskCreateRequest true "任务创建请求" +// @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskCreateResponse} "任务创建成功" +// @Failure 400 {object} network.FailureResponse "请求参数错误" +// @Failure 500 {object} network.FailureResponse "服务器内部错误" +// @Router /task/async [post] +func AsyncTaskCreateHandler(c *gin.Context) { + ctx := c.Request.Context() + var request network.AsyncTaskCreateRequest + + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "failed to unmarshal async task create request", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid request parameters", + }) + return + } + + // Validate task type + if !orm.IsValidAsyncTaskType(request.TaskType) { + logger.Error(ctx, "invalid task type", "task_type", request.TaskType) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid task type", + }) + return + } + + // Validate task parameters based on task type + if !validateTaskParams(request.TaskType, request.Params) { + logger.Error(ctx, "invalid task parameters", "task_type", request.TaskType, "params", request.Params) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid task parameters", + }) + return + } + + // Get database connection from context or use default + db := getDBFromContext(c) + if db == nil { + logger.Error(ctx, "database connection not found in context") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "database connection error", + }) + return + } + + // Create task in database + taskType := orm.AsyncTaskType(request.TaskType) + params := orm.JSONMap(request.Params) + + asyncTask, err := database.CreateAsyncTask(ctx, db, taskType, params) + if err != nil { + logger.Error(ctx, "failed to create async task in database", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to create task", + }) + return + } + + // Create task queue message + // taskQueueMsg := task.NewTaskQueueMessage(asyncTask.TaskID, task.TaskType(request.TaskType)) + + // TODO: Send task to message queue (RabbitMQ/Redis) + // This should be implemented when message queue integration is ready + // For now, we'll just log the task creation + logger.Info(ctx, "async task created successfully", "task_id", asyncTask.TaskID, "task_type", request.TaskType) + + // Return success response + c.JSON(http.StatusOK, network.SuccessResponse{ + Code: 2000, + Msg: "task created successfully", + Payload: network.AsyncTaskCreateResponse{ + TaskID: asyncTask.TaskID, + }, + }) +} + +// AsyncTaskResultQueryHandler handles querying of asynchronous task results +// @Summary 查询异步任务结果 +// @Description 根据任务ID列表查询异步任务的状态和结果 +// @Tags AsyncTask +// @Accept json +// @Produce json +// @Param task_ids query string true "任务ID列表，用逗号分隔" +// @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskResultQueryResponse} "查询成功" +// @Failure 400 {object} network.FailureResponse "请求参数错误" +// @Failure 500 {object} network.FailureResponse "服务器内部错误" +// @Router /task/async/results [get] +func AsyncTaskResultQueryHandler(c *gin.Context) { + ctx := c.Request.Context() + + // Parse task IDs from query parameter + taskIDsParam := c.Query("task_ids") + if taskIDsParam == "" { + logger.Error(ctx, "task_ids parameter is required") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "task_ids parameter is required", + }) + return + } + + // Parse comma-separated task IDs + var taskIDs []uuid.UUID + taskIDStrs := splitCommaSeparated(taskIDsParam) + for _, taskIDStr := range taskIDStrs { + taskID, err := uuid.FromString(taskIDStr) + if err != nil { + logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid task ID format", + }) + return + } + taskIDs = append(taskIDs, taskID) + } + + if len(taskIDs) == 0 { + logger.Error(ctx, "no valid task IDs provided") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "no valid task IDs provided", + }) + return + } + + // Get database connection from context or use default + db := getDBFromContext(c) + if db == nil { + logger.Error(ctx, "database connection not found in context") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "database connection error", + }) + return + } + + // Query tasks from database + asyncTasks, err := database.GetAsyncTasksByIDs(ctx, db, taskIDs) + if err != nil { + logger.Error(ctx, "failed to query async tasks from database", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to query tasks", + }) + return + } + + // Query task results from database + taskResults, err := database.GetAsyncTaskResults(ctx, db, taskIDs) + if err != nil { + logger.Error(ctx, "failed to query async task results from database", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to query task results", + }) + return + } + + // Create a map of task results for easy lookup + taskResultMap := make(map[uuid.UUID]orm.AsyncTaskResult) + for _, result := range taskResults { + taskResultMap[result.TaskID] = result + } + + // Convert to response format + var responseTasks []network.AsyncTaskResult + for _, asyncTask := range asyncTasks { + taskResult := network.AsyncTaskResult{ + TaskID: asyncTask.TaskID, + TaskType: string(asyncTask.TaskType), + Status: string(asyncTask.Status), + CreatedAt: asyncTask.CreatedAt, + FinishedAt: asyncTask.FinishedAt, + Progress: asyncTask.Progress, + } + + // Add result or error information if available + if result, exists := taskResultMap[asyncTask.TaskID]; exists { + if result.Result != nil { + taskResult.Result = map[string]any(result.Result) + } + if result.ErrorCode != nil { + taskResult.ErrorCode = result.ErrorCode + } + if result.ErrorMessage != nil { + taskResult.ErrorMessage = result.ErrorMessage + } + if result.ErrorDetail != nil { + taskResult.ErrorDetail = map[string]any(result.ErrorDetail) + } + } + + responseTasks = append(responseTasks, taskResult) + } + + // Return success response + c.JSON(http.StatusOK, network.SuccessResponse{ + Code: 2000, + Msg: "query completed", + Payload: network.AsyncTaskResultQueryResponse{ + Total: len(responseTasks), + Tasks: responseTasks, + }, + }) +} + +// AsyncTaskProgressUpdateHandler handles updating task progress (internal use, not exposed via API) +func AsyncTaskProgressUpdateHandler(c *gin.Context) { + ctx := c.Request.Context() + var request network.AsyncTaskProgressUpdate + + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "failed to unmarshal async task progress update request", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid request parameters", + }) + return + } + + // Get database connection from context or use default + db := getDBFromContext(c) + if db == nil { + logger.Error(ctx, "database connection not found in context") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "database connection error", + }) + return + } + + // Update task progress + err := database.UpdateAsyncTaskProgress(ctx, db, request.TaskID, request.Progress) + if err != nil { + logger.Error(ctx, "failed to update async task progress", "task_id", request.TaskID, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to update task progress", + }) + return + } + + c.JSON(http.StatusOK, network.SuccessResponse{ + Code: 2000, + Msg: "task progress updated successfully", + Payload: nil, + }) +} + +// AsyncTaskStatusUpdateHandler handles updating task status (internal use, not exposed via API) +func AsyncTaskStatusUpdateHandler(c *gin.Context) { + ctx := c.Request.Context() + var request network.AsyncTaskStatusUpdate + + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "failed to unmarshal async task status update request", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid request parameters", + }) + return + } + + // Validate status + validStatus := map[string]bool{ + string(orm.AsyncTaskStatusSubmitted): true, + string(orm.AsyncTaskStatusRunning): true, + string(orm.AsyncTaskStatusCompleted): true, + string(orm.AsyncTaskStatusFailed): true, + } + + if !validStatus[request.Status] { + logger.Error(ctx, "invalid task status", "status", request.Status) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid task status", + }) + return + } + + // Get database connection from context or use default + db := getDBFromContext(c) + if db == nil { + logger.Error(ctx, "database connection not found in context") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "database connection error", + }) + return + } + + // Update task status + status := orm.AsyncTaskStatus(request.Status) + err := database.UpdateAsyncTaskStatus(ctx, db, request.TaskID, status) + if err != nil { + logger.Error(ctx, "failed to update async task status", "task_id", request.TaskID, "status", request.Status, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to update task status", + }) + return + } + + // If task is completed or failed, update finished_at timestamp + if request.Status == string(orm.AsyncTaskStatusCompleted) { + err = database.CompleteAsyncTask(ctx, db, request.TaskID, request.Timestamp) + } else if request.Status == string(orm.AsyncTaskStatusFailed) { + err = database.FailAsyncTask(ctx, db, request.TaskID, request.Timestamp) + } + + if err != nil { + logger.Error(ctx, "failed to update async task completion timestamp", "task_id", request.TaskID, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to update task completion timestamp", + }) + return + } + + c.JSON(http.StatusOK, network.SuccessResponse{ + Code: 2000, + Msg: "task status updated successfully", + Payload: nil, + }) +} + +// Helper functions + +func validateTaskParams(taskType string, params map[string]any) bool { + switch taskType { + case string(orm.AsyncTaskTypeTopologyAnalysis): + return validateTopologyAnalysisParams(params) + case string(orm.AsyncTaskTypePerformanceAnalysis): + return validatePerformanceAnalysisParams(params) + case string(orm.AsyncTaskTypeEventAnalysis): + return validateEventAnalysisParams(params) + case string(orm.AsyncTaskTypeBatchImport): + return validateBatchImportParams(params) + default: + return false + } +} + +func validateTopologyAnalysisParams(params map[string]any) bool { + // Check required parameters for topology analysis + if startUUID, ok := params["start_uuid"]; !ok || startUUID == "" { + return false + } + if endUUID, ok := params["end_uuid"]; !ok || endUUID == "" { + return false + } + return true +} + +func validatePerformanceAnalysisParams(params map[string]any) bool { + // Check required parameters for performance analysis + if componentIDs, ok := params["component_ids"]; !ok { + return false + } else if ids, isSlice := componentIDs.([]interface{}); !isSlice || len(ids) == 0 { + return false + } + return true +} + +func validateEventAnalysisParams(params map[string]any) bool { + // Check required parameters for event analysis + if eventType, ok := params["event_type"]; !ok || eventType == "" { + return false + } + return true +} + +func validateBatchImportParams(params map[string]any) bool { + // Check required parameters for batch import + if filePath, ok := params["file_path"]; !ok || filePath == "" { + return false + } + return true +} + +func splitCommaSeparated(s string) []string { + var result []string + var current strings.Builder + inQuotes := false + escape := false + + for _, ch := range s { + if escape { + current.WriteRune(ch) + escape = false + continue + } + + switch ch { + case '\\': + escape = true + case '"': + inQuotes = !inQuotes + case ',': + if !inQuotes { + result = append(result, strings.TrimSpace(current.String())) + current.Reset() + } else { + current.WriteRune(ch) + } + default: + current.WriteRune(ch) + } + } + + if current.Len() > 0 { + result = append(result, strings.TrimSpace(current.String())) + } + + return result +} + +func getDBFromContext(c *gin.Context) *gorm.DB { + // Try to get database connection from context + // This should be set by middleware + if db, exists := c.Get("db"); exists { + if gormDB, ok := db.(*gorm.DB); ok { + return gormDB + } + } + + // Fallback to global database connection + // This should be implemented based on your application's database setup + // For now, return nil - actual implementation should retrieve from application context + return nil +} + +// AsyncTaskResultDetailHandler handles detailed query of a single async task result +// @Summary 查询异步任务详情 +// @Description 根据任务ID查询异步任务的详细状态和结果 +// @Tags AsyncTask +// @Accept json +// @Produce json +// @Param task_id path string true "任务ID" +// @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskResult} "查询成功" +// @Failure 400 {object} network.FailureResponse "请求参数错误" +// @Failure 404 {object} network.FailureResponse "任务不存在" +// @Failure 500 {object} network.FailureResponse "服务器内部错误" +// @Router /task/async/{task_id} [get] +func AsyncTaskResultDetailHandler(c *gin.Context) { + ctx := c.Request.Context() + + // Parse task ID from path parameter + taskIDStr := c.Param("task_id") + if taskIDStr == "" { + logger.Error(ctx, "task_id parameter is required") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "task_id parameter is required", + }) + return + } + + taskID, err := uuid.FromString(taskIDStr) + if err != nil { + logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid task ID format", + }) + return + } + + // Get database connection from context or use default + db := getDBFromContext(c) + if db == nil { + logger.Error(ctx, "database connection not found in context") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "database connection error", + }) + return + } + + // Query task from database + asyncTask, err := database.GetAsyncTaskByID(ctx, db, taskID) + if err != nil { + if err == gorm.ErrRecordNotFound { + logger.Error(ctx, "async task not found", "task_id", taskID) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusNotFound, + Msg: "task not found", + }) + return + } + logger.Error(ctx, "failed to query async task from database", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to query task", + }) + return + } + + // Query task result from database + taskResult, err := database.GetAsyncTaskResult(ctx, db, taskID) + if err != nil { + logger.Error(ctx, "failed to query async task result from database", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to query task result", + }) + return + } + + // Convert to response format + responseTask := network.AsyncTaskResult{ + TaskID: asyncTask.TaskID, + TaskType: string(asyncTask.TaskType), + Status: string(asyncTask.Status), + CreatedAt: asyncTask.CreatedAt, + FinishedAt: asyncTask.FinishedAt, + Progress: asyncTask.Progress, + } + + // Add result or error information if available + if taskResult != nil { + if taskResult.Result != nil { + responseTask.Result = map[string]any(taskResult.Result) + } + if taskResult.ErrorCode != nil { + responseTask.ErrorCode = taskResult.ErrorCode + } + if taskResult.ErrorMessage != nil { + responseTask.ErrorMessage = taskResult.ErrorMessage + } + if taskResult.ErrorDetail != nil { + responseTask.ErrorDetail = map[string]any(taskResult.ErrorDetail) + } + } + + // Return success response + c.JSON(http.StatusOK, network.SuccessResponse{ + Code: 2000, + Msg: "query completed", + Payload: responseTask, + }) +} + +// AsyncTaskCancelHandler handles cancellation of an async task +// @Summary 取消异步任务 +// @Description 取消指定ID的异步任务（如果任务尚未开始执行） +// @Tags AsyncTask +// @Accept json +// @Produce json +// @Param task_id path string true "任务ID" +// @Success 200 {object} network.SuccessResponse "任务取消成功" +// @Failure 400 {object} network.FailureResponse "请求参数错误或任务无法取消" +// @Failure 404 {object} network.FailureResponse "任务不存在" +// @Failure 500 {object} network.FailureResponse "服务器内部错误" +// @Router /task/async/{task_id}/cancel [post] +func AsyncTaskCancelHandler(c *gin.Context) { + ctx := c.Request.Context() + + // Parse task ID from path parameter + taskIDStr := c.Param("task_id") + if taskIDStr == "" { + logger.Error(ctx, "task_id parameter is required") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "task_id parameter is required", + }) + return + } + + taskID, err := uuid.FromString(taskIDStr) + if err != nil { + logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid task ID format", + }) + return + } + + // Get database connection from context or use default + db := getDBFromContext(c) + if db == nil { + logger.Error(ctx, "database connection not found in context") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "database connection error", + }) + return + } + + // Query task from database + asyncTask, err := database.GetAsyncTaskByID(ctx, db, taskID) + if err != nil { + if err == gorm.ErrRecordNotFound { + logger.Error(ctx, "async task not found", "task_id", taskID) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusNotFound, + Msg: "task not found", + }) + return + } + logger.Error(ctx, "failed to query async task from database", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to query task", + }) + return + } + + // Check if task can be cancelled (only SUBMITTED tasks can be cancelled) + if asyncTask.Status != orm.AsyncTaskStatusSubmitted { + logger.Error(ctx, "task cannot be cancelled", "task_id", taskID, "status", asyncTask.Status) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "task cannot be cancelled (already running or completed)", + }) + return + } + + // Update task status to failed with cancellation reason + timestamp := time.Now().Unix() + err = database.FailAsyncTask(ctx, db, taskID, timestamp) + if err != nil { + logger.Error(ctx, "failed to cancel async task", "task_id", taskID, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to cancel task", + }) + return + } + + // Update task result with cancellation error + err = database.UpdateAsyncTaskResultWithError(ctx, db, taskID, 40003, "task cancelled by user", orm.JSONMap{ + "cancelled_at": timestamp, + "cancelled_by": "user", + }) + if err != nil { + logger.Error(ctx, "failed to update task result with cancellation error", "task_id", taskID, "error", err) + // Continue anyway since task is already marked as failed + } + + c.JSON(http.StatusOK, network.SuccessResponse{ + Code: 2000, + Msg: "task cancelled successfully", + }) +} diff --git a/router/async_task.go b/router/async_task.go new file mode 100644 index 0000000..4b0861c --- /dev/null +++ b/router/async_task.go @@ -0,0 +1,32 @@ +// Package router provides router config +package router + +import ( + "modelRT/handler" + + "github.com/gin-gonic/gin" +) + +// registerAsyncTaskRoutes define func of register async task routes +func registerAsyncTaskRoutes(rg *gin.RouterGroup, middlewares ...gin.HandlerFunc) { + g := rg.Group("/task/") + g.Use(middlewares...) + + // Async task creation + g.POST("async", handler.AsyncTaskCreateHandler) + + // Async task result query + g.GET("async/results", handler.AsyncTaskResultQueryHandler) + + // Async task detail query + g.GET("async/:task_id", handler.AsyncTaskResultDetailHandler) + + // Async task cancellation + g.POST("async/:task_id/cancel", handler.AsyncTaskCancelHandler) + + // Internal APIs for worker updates (not exposed to external users) + internal := g.Group("internal/") + internal.Use(middlewares...) + internal.POST("async/progress", handler.AsyncTaskProgressUpdateHandler) + internal.POST("async/status", handler.AsyncTaskStatusUpdateHandler) +} \ No newline at end of file diff --git a/router/router.go b/router/router.go index 6fbc113..f242cb9 100644 --- a/router/router.go +++ b/router/router.go @@ -27,4 +27,5 @@ func RegisterRoutes(engine *gin.Engine, clientToken string) { registerDataRoutes(routeGroup) registerMonitorRoutes(routeGroup) registerComponentRoutes(routeGroup, middleware.SetTokenMiddleware(clientToken)) + registerAsyncTaskRoutes(routeGroup, middleware.SetTokenMiddleware(clientToken)) } From 7ea66e48afd0685a7a482fd3dedac3938aef2752 Mon Sep 17 00:00:00 2001 From: douxu Date: Fri, 20 Mar 2026 15:00:04 +0800 Subject: [PATCH 24/43] add code of async task system --- task/handler_factory.go | 247 ++++++++++++++++++++++ task/queue_message.go | 4 +- task/queue_producer.go | 227 +++++++++++++++++++++ task/worker.go | 441 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 918 insertions(+), 1 deletion(-) create mode 100644 task/handler_factory.go create mode 100644 task/queue_producer.go create mode 100644 task/worker.go diff --git a/task/handler_factory.go b/task/handler_factory.go new file mode 100644 index 0000000..9f6b7c2 --- /dev/null +++ b/task/handler_factory.go @@ -0,0 +1,247 @@ +// Package task provides asynchronous task processing with handler factory pattern +package task + +import ( + "context" + "fmt" + "sync" + + "modelRT/logger" + + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// TaskHandler defines the interface for task processors +type TaskHandler interface { + // Execute processes a task with the given ID and type + Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, db *gorm.DB) error + // CanHandle returns true if this handler can process the given task type + CanHandle(taskType TaskType) bool + // Name returns the name of the handler for logging and metrics + Name() string +} + +// HandlerFactory creates task handlers based on task type +type HandlerFactory struct { + handlers map[TaskType]TaskHandler + mu sync.RWMutex +} + +// NewHandlerFactory creates a new HandlerFactory +func NewHandlerFactory() *HandlerFactory { + return &HandlerFactory{ + handlers: make(map[TaskType]TaskHandler), + } +} + +// RegisterHandler registers a handler for a specific task type +func (f *HandlerFactory) RegisterHandler(taskType TaskType, handler TaskHandler) { + f.mu.Lock() + defer f.mu.Unlock() + + f.handlers[taskType] = handler + logger.Info(context.Background(), "Handler registered", + "task_type", taskType, + "handler_name", handler.Name(), + ) +} + +// GetHandler returns a handler for the given task type +func (f *HandlerFactory) GetHandler(taskType TaskType) (TaskHandler, error) { + f.mu.RLock() + handler, exists := f.handlers[taskType] + f.mu.RUnlock() + + if !exists { + return nil, fmt.Errorf("no handler registered for task type: %s", taskType) + } + + return handler, nil +} + +// CreateDefaultHandlers registers all default task handlers +func (f *HandlerFactory) CreateDefaultHandlers() { + f.RegisterHandler(TypeTopologyAnalysis, &TopologyAnalysisHandler{}) + f.RegisterHandler(TypeEventAnalysis, &EventAnalysisHandler{}) + f.RegisterHandler(TypeBatchImport, &BatchImportHandler{}) +} + +// BaseHandler provides common functionality for all task handlers +type BaseHandler struct { + name string +} + +// NewBaseHandler creates a new BaseHandler +func NewBaseHandler(name string) *BaseHandler { + return &BaseHandler{name: name} +} + +// Name returns the handler name +func (h *BaseHandler) Name() string { + return h.name +} + +// TopologyAnalysisHandler handles topology analysis tasks +type TopologyAnalysisHandler struct { + BaseHandler +} + +// NewTopologyAnalysisHandler creates a new TopologyAnalysisHandler +func NewTopologyAnalysisHandler() *TopologyAnalysisHandler { + return &TopologyAnalysisHandler{ + BaseHandler: *NewBaseHandler("topology_analysis_handler"), + } +} + +// Execute processes a topology analysis task +func (h *TopologyAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, db *gorm.DB) error { + logger.Info(ctx, "Starting topology analysis", + "task_id", taskID, + "task_type", taskType, + ) + + // TODO: Implement actual topology analysis logic + // This would typically involve: + // 1. Fetching task parameters from database + // 2. Performing topology analysis (checking for islands, shorts, etc.) + // 3. Storing results in database + // 4. Updating task status + + // Simulate work + logger.Info(ctx, "Topology analysis completed", + "task_id", taskID, + "task_type", taskType, + ) + + return nil +} + +// CanHandle returns true for topology analysis tasks +func (h *TopologyAnalysisHandler) CanHandle(taskType TaskType) bool { + return taskType == TypeTopologyAnalysis +} + +// EventAnalysisHandler handles event analysis tasks +type EventAnalysisHandler struct { + BaseHandler +} + +// NewEventAnalysisHandler creates a new EventAnalysisHandler +func NewEventAnalysisHandler() *EventAnalysisHandler { + return &EventAnalysisHandler{ + BaseHandler: *NewBaseHandler("event_analysis_handler"), + } +} + +// Execute processes an event analysis task +func (h *EventAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, db *gorm.DB) error { + logger.Info(ctx, "Starting event analysis", + "task_id", taskID, + "task_type", taskType, + ) + + // TODO: Implement actual event analysis logic + // This would typically involve: + // 1. Fetching motor and trigger information + // 2. Analyzing events within the specified duration + // 3. Generating analysis report + // 4. Storing results in database + + // Simulate work + logger.Info(ctx, "Event analysis completed", + "task_id", taskID, + "task_type", taskType, + ) + + return nil +} + +// CanHandle returns true for event analysis tasks +func (h *EventAnalysisHandler) CanHandle(taskType TaskType) bool { + return taskType == TypeEventAnalysis +} + +// BatchImportHandler handles batch import tasks +type BatchImportHandler struct { + BaseHandler +} + +// NewBatchImportHandler creates a new BatchImportHandler +func NewBatchImportHandler() *BatchImportHandler { + return &BatchImportHandler{ + BaseHandler: *NewBaseHandler("batch_import_handler"), + } +} + +// Execute processes a batch import task +func (h *BatchImportHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, db *gorm.DB) error { + logger.Info(ctx, "Starting batch import", + "task_id", taskID, + "task_type", taskType, + ) + + // TODO: Implement actual batch import logic + // This would typically involve: + // 1. Reading file from specified path + // 2. Parsing file content (CSV, Excel, etc.) + // 3. Validating and importing data into database + // 4. Generating import report + + // Simulate work + logger.Info(ctx, "Batch import completed", + "task_id", taskID, + "task_type", taskType, + ) + + return nil +} + +// CanHandle returns true for batch import tasks +func (h *BatchImportHandler) CanHandle(taskType TaskType) bool { + return taskType == TypeBatchImport +} + +// CompositeHandler can handle multiple task types by delegating to appropriate handlers +type CompositeHandler struct { + factory *HandlerFactory +} + +// NewCompositeHandler creates a new CompositeHandler +func NewCompositeHandler(factory *HandlerFactory) *CompositeHandler { + return &CompositeHandler{factory: factory} +} + +// Execute delegates task execution to the appropriate handler +func (h *CompositeHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, db *gorm.DB) error { + handler, err := h.factory.GetHandler(taskType) + if err != nil { + return fmt.Errorf("failed to get handler for task type %s: %w", taskType, err) + } + + return handler.Execute(ctx, taskID, taskType, db) +} + +// CanHandle returns true if any registered handler can handle the task type +func (h *CompositeHandler) CanHandle(taskType TaskType) bool { + _, err := h.factory.GetHandler(taskType) + return err == nil +} + +// Name returns the composite handler name +func (h *CompositeHandler) Name() string { + return "composite_handler" +} + +// DefaultHandlerFactory returns a HandlerFactory with all default handlers registered +func DefaultHandlerFactory() *HandlerFactory { + factory := NewHandlerFactory() + factory.CreateDefaultHandlers() + return factory +} + +// DefaultCompositeHandler returns a CompositeHandler with all default handlers +func DefaultCompositeHandler() TaskHandler { + factory := DefaultHandlerFactory() + return NewCompositeHandler(factory) +} \ No newline at end of file diff --git a/task/queue_message.go b/task/queue_message.go index 0c6512f..7ea0164 100644 --- a/task/queue_message.go +++ b/task/queue_message.go @@ -1,6 +1,8 @@ package task import ( + "encoding/json" + "github.com/gofrs/uuid" ) @@ -41,7 +43,7 @@ func NewTaskQueueMessageWithPriority(taskID uuid.UUID, taskType TaskType, priori // ToJSON converts the TaskQueueMessage to JSON bytes func (m *TaskQueueMessage) ToJSON() ([]byte, error) { - return []byte{}, nil // Placeholder - actual implementation would use json.Marshal + return json.Marshal(m) } // Validate checks if the TaskQueueMessage is valid diff --git a/task/queue_producer.go b/task/queue_producer.go new file mode 100644 index 0000000..b9f2df1 --- /dev/null +++ b/task/queue_producer.go @@ -0,0 +1,227 @@ +// Package task provides asynchronous task processing with RabbitMQ integration +package task + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "modelRT/config" + "modelRT/logger" + "modelRT/mq" + + "github.com/gofrs/uuid" + amqp "github.com/rabbitmq/amqp091-go" +) + +const ( + // TaskExchangeName is the name of the exchange for task routing + TaskExchangeName = "modelrt.tasks.exchange" + // TaskQueueName is the name of the main task queue + TaskQueueName = "modelrt.tasks.queue" + // TaskRoutingKey is the routing key for task messages + TaskRoutingKey = "modelrt.task" + // MaxPriority is the maximum priority level for tasks (0-10) + MaxPriority = 10 + // DefaultMessageTTL is the default time-to-live for task messages (24 hours) + DefaultMessageTTL = 24 * time.Hour +) + +// QueueProducer handles publishing tasks to RabbitMQ +type QueueProducer struct { + conn *amqp.Connection + ch *amqp.Channel +} + +// NewQueueProducer creates a new QueueProducer instance +func NewQueueProducer(ctx context.Context, cfg config.RabbitMQConfig) (*QueueProducer, error) { + // Initialize RabbitMQ connection if not already initialized + mq.InitRabbitProxy(ctx, cfg) + + conn := mq.GetConn() + if conn == nil { + return nil, fmt.Errorf("failed to get RabbitMQ connection") + } + + ch, err := conn.Channel() + if err != nil { + return nil, fmt.Errorf("failed to open channel: %w", err) + } + + producer := &QueueProducer{ + conn: conn, + ch: ch, + } + + // Declare exchange and queue + if err := producer.declareInfrastructure(); err != nil { + ch.Close() + return nil, fmt.Errorf("failed to declare infrastructure: %w", err) + } + + return producer, nil +} + +// declareInfrastructure declares the exchange, queue, and binds them +func (p *QueueProducer) declareInfrastructure() error { + // Declare durable direct exchange + err := p.ch.ExchangeDeclare( + TaskExchangeName, // name + "direct", // type + true, // durable + false, // auto-deleted + false, // internal + false, // no-wait + nil, // arguments + ) + if err != nil { + return fmt.Errorf("failed to declare exchange: %w", err) + } + + // Declare durable queue with priority support and message TTL + _, err = p.ch.QueueDeclare( + TaskQueueName, // name + true, // durable + false, // delete when unused + false, // exclusive + false, // no-wait + amqp.Table{ + "x-max-priority": MaxPriority, // support priority levels 0-10 + "x-message-ttl": DefaultMessageTTL.Milliseconds(), // message TTL + }, + ) + if err != nil { + return fmt.Errorf("failed to declare queue: %w", err) + } + + // Bind queue to exchange + err = p.ch.QueueBind( + TaskQueueName, // queue name + TaskRoutingKey, // routing key + TaskExchangeName, // exchange name + false, // no-wait + nil, // arguments + ) + if err != nil { + return fmt.Errorf("failed to bind queue: %w", err) + } + + return nil +} + +// PublishTask publishes a task message to RabbitMQ +func (p *QueueProducer) PublishTask(ctx context.Context, taskID uuid.UUID, taskType TaskType, priority int) error { + message := NewTaskQueueMessageWithPriority(taskID, taskType, priority) + + // Validate message + if !message.Validate() { + return fmt.Errorf("invalid task message: taskID=%s, taskType=%s", taskID, taskType) + } + + // Convert message to JSON + body, err := json.Marshal(message) + if err != nil { + return fmt.Errorf("failed to marshal task message: %w", err) + } + + // Prepare publishing options + publishing := amqp.Publishing{ + ContentType: "application/json", + Body: body, + DeliveryMode: amqp.Persistent, // Persistent messages survive broker restart + Timestamp: time.Now(), + Priority: uint8(priority), + Headers: amqp.Table{ + "task_id": taskID.String(), + "task_type": string(taskType), + }, + } + + // Publish to exchange + err = p.ch.PublishWithContext( + ctx, + TaskExchangeName, // exchange + TaskRoutingKey, // routing key + false, // mandatory + false, // immediate + publishing, + ) + if err != nil { + return fmt.Errorf("failed to publish task message: %w", err) + } + + logger.Info(ctx, "Task published to queue", + "task_id", taskID.String(), + "task_type", taskType, + "priority", priority, + "queue", TaskQueueName, + ) + + return nil +} + +// PublishTaskWithRetry publishes a task with retry logic +func (p *QueueProducer) PublishTaskWithRetry(ctx context.Context, taskID uuid.UUID, taskType TaskType, priority int, maxRetries int) error { + var lastErr error + for i := range maxRetries { + err := p.PublishTask(ctx, taskID, taskType, priority) + if err == nil { + return nil + } + lastErr = err + + // Exponential backoff + backoff := time.Duration(1< Date: Wed, 1 Apr 2026 17:15:33 +0800 Subject: [PATCH 25/43] implemented task queue publishing using RabbitMQ Added configuration middleware integration Added retry logic for queue publishing Added task worker initialization (main.go): Created initTaskWorker function for task worker configuration Added worker startup and shutdown logic Added CORS middleware configuration Registered config middleware --- go.mod | 42 ++++++++-------- go.sum | 91 ++++++++++++++++++----------------- handler/async_task_handler.go | 40 ++++++++++++--- main.go | 56 +++++++++++++++++++++ orm/async_task.go | 26 +++++++--- orm/async_task_result.go | 13 +++-- sql/async_task.sql | 57 ++++++++++++++++++++++ 7 files changed, 242 insertions(+), 83 deletions(-) create mode 100644 sql/async_task.sql diff --git a/go.mod b/go.mod index 99cc668..40347c5 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,8 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/RediSearch/redisearch-go/v2 v2.1.1 github.com/bitly/go-simplejson v0.5.1 - github.com/gin-gonic/gin v1.10.0 + github.com/gin-contrib/cors v1.7.6 + github.com/gin-gonic/gin v1.10.1 github.com/gofrs/uuid v4.4.0+incompatible github.com/gomodule/redigo v1.8.9 github.com/gorilla/websocket v1.5.3 @@ -16,7 +17,7 @@ require ( github.com/rabbitmq/amqp091-go v1.10.0 github.com/redis/go-redis/v9 v9.7.3 github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.4 @@ -30,25 +31,24 @@ require ( require ( github.com/BurntSushi/toml v1.4.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect - github.com/bytedance/sonic v1.12.5 // indirect - github.com/bytedance/sonic/loader v0.2.1 // indirect + github.com/bytedance/sonic v1.13.3 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.7 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/spec v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.23.0 // indirect + github.com/go-playground/validator/v10 v10.26.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect - github.com/goccy/go-json v0.10.3 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect @@ -57,7 +57,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -65,7 +65,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -75,17 +75,17 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.12 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/arch v0.12.0 // indirect - golang.org/x/crypto v0.30.0 // indirect + golang.org/x/arch v0.18.0 // indirect + golang.org/x/crypto v0.39.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.32.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/tools v0.28.0 // indirect - google.golang.org/protobuf v1.35.2 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sync v0.15.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.26.0 // indirect + golang.org/x/tools v0.33.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 2b3c5ff..3b5290c 100644 --- a/go.sum +++ b/go.sum @@ -12,16 +12,15 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/bytedance/sonic v1.12.5 h1:hoZxY8uW+mT+OpkcUWw4k0fDINtOcVavEsGfzwzFU/w= -github.com/bytedance/sonic v1.12.5/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= +github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E= -github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= -github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -33,14 +32,16 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= -github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= +github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= +github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/gin-contrib/cors v1.7.6 h1:3gQ8GMzs1Ylpf70y8bMw4fVpycXIeX1ZemuSQIsnQQY= +github.com/gin-contrib/cors v1.7.6/go.mod h1:Ulcl+xN4jel9t1Ry8vqph23a60FwH9xVLd+3ykmTjOk= github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= -github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= +github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= @@ -55,18 +56,18 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= -github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -90,8 +91,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -116,8 +117,8 @@ github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/panjf2000/ants/v2 v2.10.0 h1:zhRg1pQUtkyRiOFo2Sbqwjp0GfBNo9cUY2/Grpx1p+8= github.com/panjf2000/ants/v2 v2.10.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -150,8 +151,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= @@ -162,8 +163,8 @@ github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A= github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= -github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -173,28 +174,28 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= -golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= +golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= +golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -202,8 +203,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -211,16 +212,16 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= -golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/handler/async_task_handler.go b/handler/async_task_handler.go index 9808e33..2d7c3e9 100644 --- a/handler/async_task_handler.go +++ b/handler/async_task_handler.go @@ -6,11 +6,12 @@ import ( "strings" "time" + "modelRT/config" "modelRT/database" "modelRT/logger" "modelRT/network" "modelRT/orm" - _ "modelRT/task" + "modelRT/task" "github.com/gin-gonic/gin" "github.com/gofrs/uuid" @@ -86,12 +87,39 @@ func AsyncTaskCreateHandler(c *gin.Context) { return } - // Create task queue message - // taskQueueMsg := task.NewTaskQueueMessage(asyncTask.TaskID, task.TaskType(request.TaskType)) + // Send task to message queue + cfg, exists := c.Get("config") + if !exists { + logger.Warn(ctx, "Configuration not found in context, skipping queue publishing") + } else { + modelRTConfig := cfg.(config.ModelRTConfig) + ctx := c.Request.Context() + + // Create queue producer + producer, err := task.NewQueueProducer(ctx, modelRTConfig.RabbitMQConfig) + if err != nil { + logger.Error(ctx, "Failed to create queue producer", "error", err) + // Continue without queue publishing + } else { + defer producer.Close() + + // Publish task to queue + taskType := task.TaskType(request.TaskType) + priority := 5 // Default priority + + if err := producer.PublishTaskWithRetry(ctx, asyncTask.TaskID, taskType, priority, 3); err != nil { + logger.Error(ctx, "Failed to publish task to queue", + "task_id", asyncTask.TaskID, + "error", err) + // Log error but don't affect task creation response + } else { + logger.Info(ctx, "Task published to queue successfully", + "task_id", asyncTask.TaskID, + "queue", task.TaskQueueName) + } + } + } - // TODO: Send task to message queue (RabbitMQ/Redis) - // This should be implemented when message queue integration is ready - // For now, we'll just log the task creation logger.Info(ctx, "async task created successfully", "task_id", asyncTask.TaskID, "task_type", request.TaskType) // Return success response diff --git a/main.go b/main.go index c718793..e199889 100644 --- a/main.go +++ b/main.go @@ -12,19 +12,24 @@ import ( "os/signal" "path/filepath" "syscall" + "time" "modelRT/config" "modelRT/constants" "modelRT/database" "modelRT/diagram" "modelRT/logger" + "modelRT/middleware" "modelRT/model" "modelRT/mq" "modelRT/pool" "modelRT/real-time-data/alert" "modelRT/router" + "modelRT/task" "modelRT/util" + "github.com/gin-contrib/cors" + locker "modelRT/distributedlock" _ "modelRT/docs" @@ -66,6 +71,35 @@ var ( // // @host localhost:8080 // @BasePath /api/v1 +func initTaskWorker(ctx context.Context, config config.ModelRTConfig, db *gorm.DB) (*task.TaskWorker, error) { + // Create worker configuration + workerCfg := task.WorkerConfig{ + PoolSize: config.AsyncTaskConfig.WorkerPoolSize, + PreAlloc: true, + MaxBlockingTasks: 100, + QueueConsumerCount: config.AsyncTaskConfig.QueueConsumerCount, + PollingInterval: config.AsyncTaskConfig.HealthCheckInterval, + } + + // Create task handler factory + handlerFactory := task.NewHandlerFactory() + handlerFactory.CreateDefaultHandlers() + handler := task.DefaultCompositeHandler() + + // Create task worker + worker, err := task.NewTaskWorker(ctx, workerCfg, db, config.RabbitMQConfig, handler) + if err != nil { + return nil, fmt.Errorf("failed to create task worker: %w", err) + } + + logger.Info(ctx, "Task worker initialized", + "worker_pool_size", workerCfg.PoolSize, + "queue_consumers", workerCfg.QueueConsumerCount, + ) + + return worker, nil +} + func main() { flag.Parse() ctx := context.TODO() @@ -152,6 +186,17 @@ func main() { // init rabbitmq connection mq.InitRabbitProxy(ctx, modelRTConfig.RabbitMQConfig) + + // init async task worker + taskWorker, err := initTaskWorker(ctx, modelRTConfig, postgresDBClient) + if err != nil { + logger.Error(ctx, "Failed to initialize task worker", "error", err) + // Continue without task worker, but log warning + } else { + go taskWorker.Start() + defer taskWorker.Stop() + } + // async push event to rabbitMQ go mq.PushUpDownLimitEventToRabbitMQ(ctx, mq.MsgChan) @@ -220,6 +265,17 @@ func main() { gin.SetMode(gin.ReleaseMode) } engine := gin.New() + // 添加CORS中间件 + engine.Use(cors.New(cors.Config{ + AllowOrigins: []string{"*"}, // 或指定具体域名 + AllowMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, + AllowHeaders: []string{"Origin", "Content-Type", "Authorization"}, + ExposeHeaders: []string{"Content-Length"}, + AllowCredentials: true, + MaxAge: 12 * time.Hour, + })) + // Register configuration middleware + engine.Use(middleware.ConfigMiddleware(modelRTConfig)) router.RegisterRoutes(engine, serviceToken) // Swagger UI diff --git a/orm/async_task.go b/orm/async_task.go index 52a03c6..40e312d 100644 --- a/orm/async_task.go +++ b/orm/async_task.go @@ -35,13 +35,25 @@ const ( // AsyncTask defines the core task entity stored in database for task lifecycle tracking type AsyncTask struct { - TaskID uuid.UUID `gorm:"column:task_id;primaryKey;type:uuid;default:gen_random_uuid()"` - TaskType AsyncTaskType `gorm:"column:task_type;type:varchar(50);not null"` - Status AsyncTaskStatus `gorm:"column:status;type:varchar(20);not null;index"` - Params JSONMap `gorm:"column:params;type:jsonb"` - CreatedAt int64 `gorm:"column:created_at;not null;index"` - FinishedAt *int64 `gorm:"column:finished_at;index"` - Progress *int `gorm:"column:progress"` // 0-100, nullable + TaskID uuid.UUID `gorm:"column:task_id;primaryKey;type:uuid;default:gen_random_uuid()"` + TaskType AsyncTaskType `gorm:"column:task_type;type:varchar(50);not null;index"` + Status AsyncTaskStatus `gorm:"column:status;type:varchar(20);not null;index"` + Params JSONMap `gorm:"column:params;type:jsonb"` + CreatedAt int64 `gorm:"column:created_at;not null;index"` + FinishedAt *int64 `gorm:"column:finished_at;index"` + StartedAt *int64 `gorm:"column:started_at;index"` + ExecutionTime *int64 `gorm:"column:execution_time"` + Progress *int `gorm:"column:progress"` // 0-100, nullable + RetryCount int `gorm:"column:retry_count;default:0"` + MaxRetryCount int `gorm:"column:max_retry_count;default:3"` + NextRetryTime *int64 `gorm:"column:next_retry_time;index"` + RetryDelay int `gorm:"column:retry_delay;default:5000"` + Priority int `gorm:"column:priority;default:5;index"` + QueueName string `gorm:"column:queue_name;type:varchar(100);default:'default'"` + WorkerID *string `gorm:"column:worker_id;type:varchar(50)"` + FailureReason *string `gorm:"column:failure_reason;type:text"` + StackTrace *string `gorm:"column:stack_trace;type:text"` + CreatedBy *string `gorm:"column:created_by;type:varchar(100)"` } // TableName returns the table name for AsyncTask model diff --git a/orm/async_task_result.go b/orm/async_task_result.go index a3c6213..bbe9137 100644 --- a/orm/async_task_result.go +++ b/orm/async_task_result.go @@ -7,11 +7,16 @@ import ( // AsyncTaskResult stores computation results, separate from AsyncTask model for flexibility type AsyncTaskResult struct { - TaskID uuid.UUID `gorm:"column:task_id;primaryKey;type:uuid"` - Result JSONMap `gorm:"column:result;type:jsonb"` - ErrorCode *int `gorm:"column:error_code"` + TaskID uuid.UUID `gorm:"column:task_id;primaryKey;type:uuid"` + Result JSONMap `gorm:"column:result;type:jsonb"` + ErrorCode *int `gorm:"column:error_code"` ErrorMessage *string `gorm:"column:error_message;type:text"` - ErrorDetail JSONMap `gorm:"column:error_detail;type:jsonb"` + ErrorDetail JSONMap `gorm:"column:error_detail;type:jsonb"` + ExecutionTime int64 `gorm:"column:execution_time;not null;default:0"` + MemoryUsage *int64 `gorm:"column:memory_usage"` + CPUUsage *float64 `gorm:"column:cpu_usage"` + RetryCount int `gorm:"column:retry_count;default:0"` + CompletedAt int64 `gorm:"column:completed_at;not null"` } // TableName returns the table name for AsyncTaskResult model diff --git a/sql/async_task.sql b/sql/async_task.sql new file mode 100644 index 0000000..8b2acf9 --- /dev/null +++ b/sql/async_task.sql @@ -0,0 +1,57 @@ +-- Async task table schema migration +-- Add new columns for enhanced task tracking and retry functionality + +-- Add new columns to async_task table +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS started_at bigint NULL; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS execution_time bigint NULL; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS retry_count integer NOT NULL DEFAULT 0; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS max_retry_count integer NOT NULL DEFAULT 3; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS next_retry_time bigint NULL; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS retry_delay integer NOT NULL DEFAULT 5000; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS priority integer NOT NULL DEFAULT 5; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS queue_name varchar(100) NOT NULL DEFAULT 'default'; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS worker_id varchar(50) NULL; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS failure_reason text NULL; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS stack_trace text NULL; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS created_by varchar(100) NULL; + +-- Add new columns to async_task_result table +ALTER TABLE async_task_result ADD COLUMN IF NOT EXISTS execution_time bigint NOT NULL DEFAULT 0; +ALTER TABLE async_task_result ADD COLUMN IF NOT EXISTS memory_usage bigint NULL; +ALTER TABLE async_task_result ADD COLUMN IF NOT EXISTS cpu_usage double precision NULL; +ALTER TABLE async_task_result ADD COLUMN IF NOT EXISTS retry_count integer NOT NULL DEFAULT 0; +ALTER TABLE async_task_result ADD COLUMN IF NOT EXISTS completed_at bigint NOT NULL DEFAULT 0; + +-- Add indexes for improved query performance +CREATE INDEX IF NOT EXISTS idx_async_task_status_priority ON async_task(status, priority DESC); +CREATE INDEX IF NOT EXISTS idx_async_task_next_retry_time ON async_task(next_retry_time) WHERE status = 'FAILED'; +CREATE INDEX IF NOT EXISTS idx_async_task_created_by ON async_task(created_by); +CREATE INDEX IF NOT EXISTS idx_async_task_task_type ON async_task(task_type); +CREATE INDEX IF NOT EXISTS idx_async_task_started_at ON async_task(started_at) WHERE started_at IS NOT NULL; + +-- Update existing rows to have default values for new columns +UPDATE async_task SET priority = 5 WHERE priority IS NULL; +UPDATE async_task SET queue_name = 'default' WHERE queue_name IS NULL; +UPDATE async_task SET retry_count = 0 WHERE retry_count IS NULL; +UPDATE async_task SET max_retry_count = 3 WHERE max_retry_count IS NULL; +UPDATE async_task SET retry_delay = 5000 WHERE retry_delay IS NULL; + +-- Add comments for new columns +COMMENT ON COLUMN async_task.started_at IS 'Timestamp when task execution started (Unix epoch seconds)'; +COMMENT ON COLUMN async_task.execution_time IS 'Task execution time in milliseconds'; +COMMENT ON COLUMN async_task.retry_count IS 'Number of retry attempts for failed tasks'; +COMMENT ON COLUMN async_task.max_retry_count IS 'Maximum number of retry attempts allowed'; +COMMENT ON COLUMN async_task.next_retry_time IS 'Next retry timestamp (Unix epoch seconds)'; +COMMENT ON COLUMN async_task.retry_delay IS 'Delay between retries in milliseconds'; +COMMENT ON COLUMN async_task.priority IS 'Task priority (1-10, higher is more important)'; +COMMENT ON COLUMN async_task.queue_name IS 'Name of the queue the task belongs to'; +COMMENT ON COLUMN async_task.worker_id IS 'ID of the worker processing the task'; +COMMENT ON COLUMN async_task.failure_reason IS 'Reason for task failure'; +COMMENT ON COLUMN async_task.stack_trace IS 'Stack trace for debugging failed tasks'; +COMMENT ON COLUMN async_task.created_by IS 'User or system that created the task'; + +COMMENT ON COLUMN async_task_result.execution_time IS 'Total execution time in milliseconds'; +COMMENT ON COLUMN async_task_result.memory_usage IS 'Memory usage in bytes'; +COMMENT ON COLUMN async_task_result.cpu_usage IS 'CPU usage percentage'; +COMMENT ON COLUMN async_task_result.retry_count IS 'Number of retries before success'; +COMMENT ON COLUMN async_task_result.completed_at IS 'Timestamp when task completed (Unix epoch seconds)'; \ No newline at end of file From f8c0951a1399961938e53f9dbc50e8463fc2925f Mon Sep 17 00:00:00 2001 From: douxu Date: Fri, 3 Apr 2026 10:07:43 +0800 Subject: [PATCH 26/43] Extend async task system with database integration and retry management - Add AsyncTaskConfig to config structure - Create database operations for task state management (async_task_extended.go) - Add configuration middleware for Gin context - Extract task worker initialization to separate file (initializer.go) - Implement retry strategies with exponential backoff (retry_manager.go) - Add retry queue for failed task scheduling (retry_queue.go) - Enhance worker metrics with detailed per-task-type tracking - Integrate database operations into task worker for status updates - Add comprehensive metrics logging system --- config/config.go | 12 ++ database/async_task_extended.go | 227 ++++++++++++++++++++++++++++++++ main.go | 30 +---- middleware/config_middleware.go | 15 +++ task/initializer.go | 41 ++++++ task/metrics_logger.go | 157 ++++++++++++++++++++++ task/retry_manager.go | 219 ++++++++++++++++++++++++++++++ task/retry_queue.go | 187 ++++++++++++++++++++++++++ task/worker.go | 208 ++++++++++++++++++++++++++--- 9 files changed, 1048 insertions(+), 48 deletions(-) create mode 100644 database/async_task_extended.go create mode 100644 middleware/config_middleware.go create mode 100644 task/initializer.go create mode 100644 task/metrics_logger.go create mode 100644 task/retry_manager.go create mode 100644 task/retry_queue.go diff --git a/config/config.go b/config/config.go index 3c09187..1425f45 100644 --- a/config/config.go +++ b/config/config.go @@ -3,6 +3,7 @@ package config import ( "fmt" + "time" "github.com/spf13/viper" ) @@ -91,6 +92,16 @@ type DataRTConfig struct { Method string `mapstructure:"polling_api_method"` } +// AsyncTaskConfig define config struct of asynchronous task system +type AsyncTaskConfig struct { + WorkerPoolSize int `mapstructure:"worker_pool_size"` + QueueConsumerCount int `mapstructure:"queue_consumer_count"` + MaxRetryCount int `mapstructure:"max_retry_count"` + RetryInitialDelay time.Duration `mapstructure:"retry_initial_delay"` + RetryMaxDelay time.Duration `mapstructure:"retry_max_delay"` + HealthCheckInterval time.Duration `mapstructure:"health_check_interval"` +} + // ModelRTConfig define config struct of model runtime server type ModelRTConfig struct { BaseConfig `mapstructure:"base"` @@ -103,6 +114,7 @@ type ModelRTConfig struct { DataRTConfig `mapstructure:"dataRT"` LockerRedisConfig RedisConfig `mapstructure:"locker_redis"` StorageRedisConfig RedisConfig `mapstructure:"storage_redis"` + AsyncTaskConfig AsyncTaskConfig `mapstructure:"async_task"` PostgresDBURI string `mapstructure:"-"` } diff --git a/database/async_task_extended.go b/database/async_task_extended.go new file mode 100644 index 0000000..ca94b42 --- /dev/null +++ b/database/async_task_extended.go @@ -0,0 +1,227 @@ +// Package database define database operation functions +package database + +import ( + "context" + "time" + + "modelRT/orm" + + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// UpdateTaskStarted updates task start time and status to running +func UpdateTaskStarted(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, startedAt int64) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "status": orm.AsyncTaskStatusRunning, + "started_at": startedAt, + }) + + return result.Error +} + +// UpdateTaskRetryInfo updates task retry information +func UpdateTaskRetryInfo(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, retryCount int, nextRetryTime int64) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + updateData := map[string]any{ + "retry_count": retryCount, + } + if nextRetryTime <= 0 { + updateData["next_retry_time"] = nil + } else { + updateData["next_retry_time"] = nextRetryTime + } + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Updates(updateData) + + return result.Error +} + +// UpdateTaskErrorInfo updates task error information +func UpdateTaskErrorInfo(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, errorMsg, stackTrace string) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "failure_reason": errorMsg, + "stack_trace": stackTrace, + }) + + return result.Error +} + +// UpdateTaskExecutionTime updates task execution time +func UpdateTaskExecutionTime(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, executionTime int64) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("execution_time", executionTime) + + return result.Error +} + +// UpdateTaskWorkerID updates the worker ID that is processing the task +func UpdateTaskWorkerID(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, workerID string) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("worker_id", workerID) + + return result.Error +} + +// UpdateTaskPriority updates task priority +func UpdateTaskPriority(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, priority int) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("priority", priority) + + return result.Error +} + +// UpdateTaskQueueName updates task queue name +func UpdateTaskQueueName(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, queueName string) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("queue_name", queueName) + + return result.Error +} + +// UpdateTaskCreatedBy updates task creator information +func UpdateTaskCreatedBy(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, createdBy string) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("created_by", createdBy) + + return result.Error +} + +// UpdateTaskResultWithMetrics updates task result with execution metrics +func UpdateTaskResultWithMetrics(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, executionTime int64, memoryUsage *int64, cpuUsage *float64, retryCount int, completedAt int64) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTaskResult{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "execution_time": executionTime, + "memory_usage": memoryUsage, + "cpu_usage": cpuUsage, + "retry_count": retryCount, + "completed_at": completedAt, + }) + + return result.Error +} + +// GetTasksForRetry retrieves tasks that are due for retry +func GetTasksForRetry(ctx context.Context, tx *gorm.DB, limit int) ([]orm.AsyncTask, error) { + var tasks []orm.AsyncTask + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + now := time.Now().Unix() + result := tx.WithContext(cancelCtx). + Where("status = ? AND next_retry_time IS NOT NULL AND next_retry_time <= ?", orm.AsyncTaskStatusFailed, now). + Order("next_retry_time ASC"). + Limit(limit). + Find(&tasks) + + if result.Error != nil { + return nil, result.Error + } + + return tasks, nil +} + +// GetTasksByPriority retrieves tasks by priority order +func GetTasksByPriority(ctx context.Context, tx *gorm.DB, status orm.AsyncTaskStatus, limit int) ([]orm.AsyncTask, error) { + var tasks []orm.AsyncTask + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("status = ?", status). + Order("priority DESC, created_at ASC"). + Limit(limit). + Find(&tasks) + + if result.Error != nil { + return nil, result.Error + } + + return tasks, nil +} + +// GetTasksByWorkerID retrieves tasks being processed by a specific worker +func GetTasksByWorkerID(ctx context.Context, tx *gorm.DB, workerID string) ([]orm.AsyncTask, error) { + var tasks []orm.AsyncTask + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("worker_id = ? AND status = ?", workerID, orm.AsyncTaskStatusRunning). + Find(&tasks) + + if result.Error != nil { + return nil, result.Error + } + + return tasks, nil +} + +// CleanupStaleTasks marks tasks as failed if they have been running for too long +func CleanupStaleTasks(ctx context.Context, tx *gorm.DB, timeoutSeconds int64) (int64, error) { + cancelCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + threshold := time.Now().Unix() - timeoutSeconds + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("status = ? AND started_at IS NOT NULL AND started_at < ?", orm.AsyncTaskStatusRunning, threshold). + Updates(map[string]any{ + "status": orm.AsyncTaskStatusFailed, + "failure_reason": "task timeout", + "finished_at": time.Now().Unix(), + }) + + return result.RowsAffected, result.Error +} \ No newline at end of file diff --git a/main.go b/main.go index e199889..198bec0 100644 --- a/main.go +++ b/main.go @@ -71,34 +71,6 @@ var ( // // @host localhost:8080 // @BasePath /api/v1 -func initTaskWorker(ctx context.Context, config config.ModelRTConfig, db *gorm.DB) (*task.TaskWorker, error) { - // Create worker configuration - workerCfg := task.WorkerConfig{ - PoolSize: config.AsyncTaskConfig.WorkerPoolSize, - PreAlloc: true, - MaxBlockingTasks: 100, - QueueConsumerCount: config.AsyncTaskConfig.QueueConsumerCount, - PollingInterval: config.AsyncTaskConfig.HealthCheckInterval, - } - - // Create task handler factory - handlerFactory := task.NewHandlerFactory() - handlerFactory.CreateDefaultHandlers() - handler := task.DefaultCompositeHandler() - - // Create task worker - worker, err := task.NewTaskWorker(ctx, workerCfg, db, config.RabbitMQConfig, handler) - if err != nil { - return nil, fmt.Errorf("failed to create task worker: %w", err) - } - - logger.Info(ctx, "Task worker initialized", - "worker_pool_size", workerCfg.PoolSize, - "queue_consumers", workerCfg.QueueConsumerCount, - ) - - return worker, nil -} func main() { flag.Parse() @@ -188,7 +160,7 @@ func main() { mq.InitRabbitProxy(ctx, modelRTConfig.RabbitMQConfig) // init async task worker - taskWorker, err := initTaskWorker(ctx, modelRTConfig, postgresDBClient) + taskWorker, err := task.InitTaskWorker(ctx, modelRTConfig, postgresDBClient) if err != nil { logger.Error(ctx, "Failed to initialize task worker", "error", err) // Continue without task worker, but log warning diff --git a/middleware/config_middleware.go b/middleware/config_middleware.go new file mode 100644 index 0000000..ff56995 --- /dev/null +++ b/middleware/config_middleware.go @@ -0,0 +1,15 @@ +package middleware + +import ( + "modelRT/config" + + "github.com/gin-gonic/gin" +) + +// ConfigMiddleware 将全局配置注入到Gin上下文中 +func ConfigMiddleware(modelRTConfig config.ModelRTConfig) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("config", modelRTConfig) + c.Next() + } +} \ No newline at end of file diff --git a/task/initializer.go b/task/initializer.go new file mode 100644 index 0000000..de75cea --- /dev/null +++ b/task/initializer.go @@ -0,0 +1,41 @@ +// Package task provides asynchronous task processing with worker pools +package task + +import ( + "context" + "fmt" + + "modelRT/config" + "modelRT/logger" + "gorm.io/gorm" +) + +// InitTaskWorker initializes a task worker with the given configuration and database connection +func InitTaskWorker(ctx context.Context, config config.ModelRTConfig, db *gorm.DB) (*TaskWorker, error) { + // Create worker configuration + workerCfg := WorkerConfig{ + PoolSize: config.AsyncTaskConfig.WorkerPoolSize, + PreAlloc: true, + MaxBlockingTasks: 100, + QueueConsumerCount: config.AsyncTaskConfig.QueueConsumerCount, + PollingInterval: config.AsyncTaskConfig.HealthCheckInterval, + } + + // Create task handler factory + handlerFactory := NewHandlerFactory() + handlerFactory.CreateDefaultHandlers() + handler := DefaultCompositeHandler() + + // Create task worker + worker, err := NewTaskWorker(ctx, workerCfg, db, config.RabbitMQConfig, handler) + if err != nil { + return nil, fmt.Errorf("failed to create task worker: %w", err) + } + + logger.Info(ctx, "Task worker initialized", + "worker_pool_size", workerCfg.PoolSize, + "queue_consumers", workerCfg.QueueConsumerCount, + ) + + return worker, nil +} \ No newline at end of file diff --git a/task/metrics_logger.go b/task/metrics_logger.go new file mode 100644 index 0000000..d8c0675 --- /dev/null +++ b/task/metrics_logger.go @@ -0,0 +1,157 @@ +// Package task provides metrics logging for asynchronous task system +package task + +import ( + "context" + "runtime" + "time" + + "modelRT/logger" +) + +// MetricsLogger logs task system metrics using the existing logging system +type MetricsLogger struct { + ctx context.Context +} + +// NewMetricsLogger creates a new MetricsLogger +func NewMetricsLogger(ctx context.Context) *MetricsLogger { + return &MetricsLogger{ctx: ctx} +} + +// LogTaskMetrics records task processing metrics +func (m *MetricsLogger) LogTaskMetrics(taskType TaskType, status string, processingTime time.Duration, retryCount int) { + logger.Info(m.ctx, "Task metrics", + "task_type", taskType, + "status", status, + "processing_time_ms", processingTime.Milliseconds(), + "retry_count", retryCount, + "metric_type", "task_processing", + "timestamp", time.Now().Unix(), + ) +} + +// LogQueueMetrics records queue metrics +func (m *MetricsLogger) LogQueueMetrics(queueDepth int, queueLatency time.Duration) { + logger.Info(m.ctx, "Queue metrics", + "queue_depth", queueDepth, + "queue_latency_ms", queueLatency.Milliseconds(), + "metric_type", "queue", + "timestamp", time.Now().Unix(), + ) +} + +// LogWorkerMetrics records worker metrics +func (m *MetricsLogger) LogWorkerMetrics(activeWorkers, idleWorkers, totalWorkers int, memoryUsage uint64, cpuLoad float64) { + logger.Info(m.ctx, "Worker metrics", + "active_workers", activeWorkers, + "idle_workers", idleWorkers, + "total_workers", totalWorkers, + "memory_usage_mb", memoryUsage/(1024*1024), + "cpu_load_percent", cpuLoad, + "metric_type", "worker", + "timestamp", time.Now().Unix(), + ) +} + +// LogRetryMetrics records retry metrics +func (m *MetricsLogger) LogRetryMetrics(taskType TaskType, retryCount int, success bool, delay time.Duration) { + logger.Info(m.ctx, "Retry metrics", + "task_type", taskType, + "retry_count", retryCount, + "retry_success", success, + "retry_delay_ms", delay.Milliseconds(), + "metric_type", "retry", + "timestamp", time.Now().Unix(), + ) +} + +// LogSystemMetrics records system-level metrics (memory, CPU, goroutines) +func (m *MetricsLogger) LogSystemMetrics() { + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + logger.Info(m.ctx, "System metrics", + "metric_type", "system", + "timestamp", time.Now().Unix(), + "goroutines", runtime.NumGoroutine(), + "memory_alloc_mb", memStats.Alloc/(1024*1024), + "memory_total_alloc_mb", memStats.TotalAlloc/(1024*1024), + "memory_sys_mb", memStats.Sys/(1024*1024), + "memory_heap_alloc_mb", memStats.HeapAlloc/(1024*1024), + "memory_heap_sys_mb", memStats.HeapSys/(1024*1024), + "memory_heap_inuse_mb", memStats.HeapInuse/(1024*1024), + "gc_pause_total_ns", memStats.PauseTotalNs, + "num_gc", memStats.NumGC, + ) +} + +// LogTaskCompletionMetrics records detailed task completion metrics +func (m *MetricsLogger) LogTaskCompletionMetrics(taskID, taskType, status string, startTime, endTime time.Time, retryCount int, errorMsg string) { + duration := endTime.Sub(startTime) + + logger.Info(m.ctx, "Task completion metrics", + "metric_type", "task_completion", + "timestamp", time.Now().Unix(), + "task_id", taskID, + "task_type", taskType, + "status", status, + "duration_ms", duration.Milliseconds(), + "start_time", startTime.Unix(), + "end_time", endTime.Unix(), + "retry_count", retryCount, + "has_error", errorMsg != "", + "error_msg", errorMsg, + ) +} + +// LogHealthCheckMetrics records health check metrics +func (m *MetricsLogger) LogHealthCheckMetrics(healthy bool, checkDuration time.Duration, components map[string]bool) { + logger.Info(m.ctx, "Health check metrics", + "metric_type", "health_check", + "timestamp", time.Now().Unix(), + "healthy", healthy, + "check_duration_ms", checkDuration.Milliseconds(), + "components", components, + ) +} + +// PeriodicMetricsLogger periodically logs system and worker metrics +type PeriodicMetricsLogger struct { + ctx context.Context + interval time.Duration + stopChan chan struct{} + metricsLog *MetricsLogger +} + +// NewPeriodicMetricsLogger creates a new PeriodicMetricsLogger +func NewPeriodicMetricsLogger(ctx context.Context, interval time.Duration) *PeriodicMetricsLogger { + return &PeriodicMetricsLogger{ + ctx: ctx, + interval: interval, + stopChan: make(chan struct{}), + metricsLog: NewMetricsLogger(ctx), + } +} + +// Start begins periodic metrics logging +func (p *PeriodicMetricsLogger) Start() { + go func() { + ticker := time.NewTicker(p.interval) + defer ticker.Stop() + + for { + select { + case <-p.stopChan: + return + case <-ticker.C: + p.metricsLog.LogSystemMetrics() + } + } + }() +} + +// Stop stops periodic metrics logging +func (p *PeriodicMetricsLogger) Stop() { + close(p.stopChan) +} \ No newline at end of file diff --git a/task/retry_manager.go b/task/retry_manager.go new file mode 100644 index 0000000..ddd7db5 --- /dev/null +++ b/task/retry_manager.go @@ -0,0 +1,219 @@ +// Package task provides retry strategies for failed asynchronous tasks +package task + +import ( + "context" + "math" + "math/rand" + "strings" + "time" + + "modelRT/logger" +) + +// RetryStrategy defines the interface for task retry strategies +type RetryStrategy interface { + // ShouldRetry determines if a task should be retried and returns the delay before next retry + ShouldRetry(ctx context.Context, taskID string, retryCount int, lastError error) (bool, time.Duration) + // GetMaxRetries returns the maximum number of retry attempts + GetMaxRetries() int +} + +// ExponentialBackoffRetry implements exponential backoff with jitter retry strategy +type ExponentialBackoffRetry struct { + MaxRetries int + InitialDelay time.Duration + MaxDelay time.Duration + RandomFactor float64 // Jitter factor to avoid thundering herd problem +} + +// NewExponentialBackoffRetry creates a new exponential backoff retry strategy +func NewExponentialBackoffRetry(maxRetries int, initialDelay, maxDelay time.Duration, randomFactor float64) *ExponentialBackoffRetry { + if maxRetries < 0 { + maxRetries = 0 + } + if initialDelay <= 0 { + initialDelay = 1 * time.Second + } + if maxDelay <= 0 { + maxDelay = 5 * time.Minute + } + if randomFactor < 0 { + randomFactor = 0 + } + if randomFactor > 1 { + randomFactor = 1 + } + + return &ExponentialBackoffRetry{ + MaxRetries: maxRetries, + InitialDelay: initialDelay, + MaxDelay: maxDelay, + RandomFactor: randomFactor, + } +} + +// ShouldRetry implements exponential backoff with jitter +func (s *ExponentialBackoffRetry) ShouldRetry(ctx context.Context, taskID string, retryCount int, lastError error) (bool, time.Duration) { + if retryCount >= s.MaxRetries { + logger.Info(ctx, "Task reached maximum retry count", + "task_id", taskID, + "retry_count", retryCount, + "max_retries", s.MaxRetries, + "last_error", lastError, + ) + return false, 0 + } + + // Calculate exponential backoff: initialDelay * 2^retryCount + delay := s.InitialDelay * time.Duration(math.Pow(2, float64(retryCount))) + + // Apply maximum delay cap + if delay > s.MaxDelay { + delay = s.MaxDelay + } + + // Add jitter to avoid thundering herd + if s.RandomFactor > 0 { + jitter := rand.Float64() * s.RandomFactor * float64(delay) + // Randomly add or subtract jitter + if rand.Intn(2) == 0 { + delay += time.Duration(jitter) + } else { + delay -= time.Duration(jitter) + } + // Ensure delay doesn't go below initial delay + if delay < s.InitialDelay { + delay = s.InitialDelay + } + } + + logger.Info(ctx, "Task will be retried", + "task_id", taskID, + "retry_count", retryCount, + "next_retry_in", delay, + "max_retries", s.MaxRetries, + ) + + return true, delay +} + +// GetMaxRetries returns the maximum number of retry attempts +func (s *ExponentialBackoffRetry) GetMaxRetries() int { + return s.MaxRetries +} + +// FixedDelayRetry implements fixed delay retry strategy +type FixedDelayRetry struct { + MaxRetries int + Delay time.Duration + RandomFactor float64 +} + +// NewFixedDelayRetry creates a new fixed delay retry strategy +func NewFixedDelayRetry(maxRetries int, delay time.Duration, randomFactor float64) *FixedDelayRetry { + if maxRetries < 0 { + maxRetries = 0 + } + if delay <= 0 { + delay = 5 * time.Second + } + + return &FixedDelayRetry{ + MaxRetries: maxRetries, + Delay: delay, + RandomFactor: randomFactor, + } +} + +// ShouldRetry implements fixed delay with optional jitter +func (s *FixedDelayRetry) ShouldRetry(ctx context.Context, taskID string, retryCount int, lastError error) (bool, time.Duration) { + if retryCount >= s.MaxRetries { + return false, 0 + } + + delay := s.Delay + + // Add jitter if random factor is specified + if s.RandomFactor > 0 { + jitter := rand.Float64() * s.RandomFactor * float64(delay) + if rand.Intn(2) == 0 { + delay += time.Duration(jitter) + } else { + delay -= time.Duration(jitter) + } + // Ensure positive delay + if delay <= 0 { + delay = s.Delay + } + } + + return true, delay +} + +// GetMaxRetries returns the maximum number of retry attempts +func (s *FixedDelayRetry) GetMaxRetries() int { + return s.MaxRetries +} + +// NoRetryStrategy implements a strategy that never retries +type NoRetryStrategy struct{} + +// NewNoRetryStrategy creates a new no-retry strategy +func NewNoRetryStrategy() *NoRetryStrategy { + return &NoRetryStrategy{} +} + +// ShouldRetry always returns false +func (s *NoRetryStrategy) ShouldRetry(ctx context.Context, taskID string, retryCount int, lastError error) (bool, time.Duration) { + return false, 0 +} + +// GetMaxRetries returns 0 +func (s *NoRetryStrategy) GetMaxRetries() int { + return 0 +} + +// DefaultRetryStrategy returns the default retry strategy (exponential backoff) +func DefaultRetryStrategy() RetryStrategy { + return NewExponentialBackoffRetry( + 3, // max retries + 1*time.Second, // initial delay + 5*time.Minute, // max delay + 0.1, // random factor (10% jitter) + ) +} + +// IsRetryableError checks if an error is retryable based on common patterns +func IsRetryableError(err error) bool { + if err == nil { + return false + } + + errorMsg := err.Error() + + // Check for transient errors that are typically retryable + retryablePatterns := []string{ + "timeout", + "deadline exceeded", + "temporary", + "busy", + "connection refused", + "connection reset", + "network", + "too many connections", + "resource temporarily unavailable", + "rate limit", + "throttle", + "server unavailable", + "service unavailable", + } + + for _, pattern := range retryablePatterns { + if strings.Contains(strings.ToLower(errorMsg), pattern) { + return true + } + } + + return false +} \ No newline at end of file diff --git a/task/retry_queue.go b/task/retry_queue.go new file mode 100644 index 0000000..c602f66 --- /dev/null +++ b/task/retry_queue.go @@ -0,0 +1,187 @@ +// Package task provides retry queue management for failed asynchronous tasks +package task + +import ( + "context" + "time" + + "modelRT/database" + "modelRT/logger" + + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// RetryQueue manages scheduling and execution of task retries +type RetryQueue struct { + db *gorm.DB + producer *QueueProducer + strategy RetryStrategy +} + +// NewRetryQueue creates a new RetryQueue instance +func NewRetryQueue(db *gorm.DB, producer *QueueProducer, strategy RetryStrategy) *RetryQueue { + if strategy == nil { + strategy = DefaultRetryStrategy() + } + + return &RetryQueue{ + db: db, + producer: producer, + strategy: strategy, + } +} + +// ScheduleRetry schedules a failed task for retry based on retry strategy +func (q *RetryQueue) ScheduleRetry(ctx context.Context, taskID uuid.UUID, taskType TaskType, retryCount int, lastError error) error { + // Check if task should be retried + shouldRetry, delay := q.strategy.ShouldRetry(ctx, taskID.String(), retryCount, lastError) + if !shouldRetry { + // Mark task as permanently failed + logger.Info(ctx, "Task will not be retried, marking as failed", + "task_id", taskID, + "retry_count", retryCount, + "max_retries", q.strategy.GetMaxRetries(), + "last_error", lastError, + ) + return database.FailAsyncTask(ctx, q.db, taskID, time.Now().Unix()) + } + + // Calculate next retry time + nextRetryTime := time.Now().Add(delay).Unix() + + // Update task retry information in database + err := q.db.Transaction(func(tx *gorm.DB) error { + if err := database.UpdateTaskRetryInfo(ctx, tx, taskID, retryCount+1, nextRetryTime); err != nil { + return err + } + + // Update error information + errorMsg := "" + if lastError != nil { + errorMsg = lastError.Error() + } + if err := database.UpdateTaskErrorInfo(ctx, tx, taskID, errorMsg, ""); err != nil { + // Log but don't fail the whole retry scheduling + logger.Warn(ctx, "Failed to update task error info", + "task_id", taskID, + "error", err, + ) + } + + // Task will be picked up by ProcessRetryQueue when next_retry_time is reached + return nil + }) + + if err != nil { + logger.Error(ctx, "Failed to schedule task retry", + "task_id", taskID, + "task_type", taskType, + "retry_count", retryCount, + "delay", delay, + "error", err, + ) + return err + } + + logger.Info(ctx, "Task scheduled for retry", + "task_id", taskID, + "task_type", taskType, + "retry_count", retryCount+1, + "next_retry_in", delay, + "next_retry_time", time.Unix(nextRetryTime, 0).Format(time.RFC3339), + ) + + return nil +} + +// ProcessRetryQueue processes tasks that are due for retry +func (q *RetryQueue) ProcessRetryQueue(ctx context.Context, batchSize int) error { + // Get tasks due for retry + tasks, err := database.GetTasksForRetry(ctx, q.db, batchSize) + if err != nil { + logger.Error(ctx, "Failed to get tasks for retry", "error", err) + return err + } + + if len(tasks) == 0 { + return nil + } + + logger.Info(ctx, "Processing retry queue", + "task_count", len(tasks), + "batch_size", batchSize, + ) + + for _, task := range tasks { + select { + case <-ctx.Done(): + return ctx.Err() + default: + // Publish task to queue for immediate processing + taskType := TaskType(task.TaskType) + if err := q.producer.PublishTask(ctx, task.TaskID, taskType, task.Priority); err != nil { + logger.Error(ctx, "Failed to publish retry task to queue", + "task_id", task.TaskID, + "task_type", taskType, + "error", err, + ) + // Continue with other tasks + continue + } + + // Update task status back to submitted + if err := database.UpdateAsyncTaskStatus(ctx, q.db, task.TaskID, "SUBMITTED"); err != nil { + logger.Warn(ctx, "Failed to update retry task status", + "task_id", task.TaskID, + "error", err, + ) + } + + // Clear next retry time since task is being retried now + if err := database.UpdateTaskRetryInfo(ctx, q.db, task.TaskID, task.RetryCount, 0); err != nil { + logger.Warn(ctx, "Failed to clear next retry time", + "task_id", task.TaskID, + "error", err, + ) + } + + logger.Info(ctx, "Retry task resubmitted", + "task_id", task.TaskID, + "task_type", taskType, + "retry_count", task.RetryCount, + ) + } + } + + return nil +} + +// StartRetryScheduler starts a background goroutine to periodically process retry queue +func (q *RetryQueue) StartRetryScheduler(ctx context.Context, interval time.Duration, batchSize int) { + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + logger.Info(ctx, "Retry scheduler stopping") + return + case <-ticker.C: + if err := q.ProcessRetryQueue(ctx, batchSize); err != nil { + logger.Error(ctx, "Error processing retry queue", "error", err) + } + } + } + }() +} + +// GetRetryStats returns statistics about retry queue +func (q *RetryQueue) GetRetryStats(ctx context.Context) (int, error) { + tasks, err := database.GetTasksForRetry(ctx, q.db, 1000) // Large limit to count + if err != nil { + return 0, err + } + return len(tasks), nil +} \ No newline at end of file diff --git a/task/worker.go b/task/worker.go index c33198a..8388f8e 100644 --- a/task/worker.go +++ b/task/worker.go @@ -9,8 +9,10 @@ import ( "time" "modelRT/config" + "modelRT/database" "modelRT/logger" "modelRT/mq" + "modelRT/orm" "github.com/gofrs/uuid" "github.com/panjf2000/ants/v2" @@ -51,6 +53,7 @@ type TaskWorker struct { conn *amqp.Connection ch *amqp.Channel handler TaskHandler + retryQueue *RetryQueue stopChan chan struct{} wg sync.WaitGroup ctx context.Context @@ -60,12 +63,34 @@ type TaskWorker struct { // WorkerMetrics holds metrics for the worker pool type WorkerMetrics struct { - TasksProcessed int64 - TasksFailed int64 + // Task statistics by type + TasksProcessed map[TaskType]int64 + TasksFailed map[TaskType]int64 + TasksSuccess map[TaskType]int64 + ProcessingTime map[TaskType]time.Duration + + // Aggregate counters (maintained for backward compatibility) + TotalProcessed int64 + TotalFailed int64 + TotalSuccess int64 TasksInProgress int32 + + // Queue and latency metrics QueueDepth int + QueueLatency time.Duration + + // Worker resource metrics WorkersActive int WorkersIdle int + MemoryUsage uint64 + CPULoad float64 + + // Time window metrics + LastMinuteRate float64 + Last5MinutesRate float64 + LastHourRate float64 + + // Health and timing LastHealthCheck time.Time mu sync.RWMutex } @@ -133,6 +158,10 @@ func NewTaskWorker(ctx context.Context, cfg WorkerConfig, db *gorm.DB, rabbitCfg ctx: ctxWithCancel, cancel: cancel, metrics: &WorkerMetrics{ + TasksProcessed: make(map[TaskType]int64), + TasksFailed: make(map[TaskType]int64), + TasksSuccess: make(map[TaskType]int64), + ProcessingTime: make(map[TaskType]time.Duration), LastHealthCheck: time.Now(), }, } @@ -232,7 +261,7 @@ func (w *TaskWorker) handleMessage(msg amqp.Delivery) { logger.Error(ctx, "Failed to unmarshal task message", "error", err) msg.Nack(false, false) // Reject without requeue w.metrics.mu.Lock() - w.metrics.TasksFailed++ + w.metrics.TotalFailed++ w.metrics.mu.Unlock() return } @@ -245,7 +274,9 @@ func (w *TaskWorker) handleMessage(msg amqp.Delivery) { ) msg.Nack(false, false) // Reject without requeue w.metrics.mu.Lock() - w.metrics.TasksFailed++ + w.metrics.TotalFailed++ + // Also update per-task-type failure count + w.metrics.TasksFailed[taskMsg.TaskType]++ w.metrics.mu.Unlock() return } @@ -261,7 +292,8 @@ func (w *TaskWorker) handleMessage(msg amqp.Delivery) { logger.Error(ctx, "Failed to update task status", "error", err) msg.Nack(false, true) // Reject with requeue w.metrics.mu.Lock() - w.metrics.TasksFailed++ + w.metrics.TotalFailed++ + w.metrics.TasksFailed[taskMsg.TaskType]++ w.metrics.mu.Unlock() return } @@ -287,7 +319,8 @@ func (w *TaskWorker) handleMessage(msg amqp.Delivery) { // Ack message even if task failed (we don't want to retry indefinitely) msg.Ack(false) w.metrics.mu.Lock() - w.metrics.TasksFailed++ + w.metrics.TotalFailed++ + w.metrics.TasksFailed[taskMsg.TaskType]++ w.metrics.mu.Unlock() return } @@ -308,18 +341,74 @@ func (w *TaskWorker) handleMessage(msg amqp.Delivery) { ) w.metrics.mu.Lock() - w.metrics.TasksProcessed++ + w.metrics.TotalProcessed++ + w.metrics.TasksProcessed[taskMsg.TaskType]++ + w.metrics.TasksSuccess[taskMsg.TaskType]++ + w.metrics.ProcessingTime[taskMsg.TaskType] += processingTime w.metrics.mu.Unlock() } // updateTaskStatus updates the status of a task in the database func (w *TaskWorker) updateTaskStatus(ctx context.Context, taskID uuid.UUID, status TaskStatus) error { - // This is a simplified version. In a real implementation, you would: - // 1. Have a proper task table/model - // 2. Update the task record with status and timestamps + // Convert TaskStatus to orm.AsyncTaskStatus + var ormStatus orm.AsyncTaskStatus + switch status { + case StatusPending: + ormStatus = orm.AsyncTaskStatusSubmitted + case StatusRunning: + ormStatus = orm.AsyncTaskStatusRunning + case StatusCompleted: + ormStatus = orm.AsyncTaskStatusCompleted + case StatusFailed: + ormStatus = orm.AsyncTaskStatusFailed + default: + return fmt.Errorf("unknown task status: %s", status) + } - // For now, we'll log the update - logger.Debug(ctx, "Updating task status", + // Update task status in database + err := database.UpdateAsyncTaskStatus(ctx, w.db, taskID, ormStatus) + if err != nil { + logger.Error(ctx, "Failed to update task status in database", + "task_id", taskID, + "status", status, + "error", err, + ) + return err + } + + // If status is running, update started_at timestamp + if status == StatusRunning { + startedAt := time.Now().Unix() + if err := database.UpdateTaskStarted(ctx, w.db, taskID, startedAt); err != nil { + logger.Warn(ctx, "Failed to update task start time", + "task_id", taskID, + "error", err, + ) + // Continue despite error + } + } + + // If status is completed or failed, update finished_at timestamp + if status == StatusCompleted || status == StatusFailed { + finishedAt := time.Now().Unix() + if status == StatusCompleted { + if err := database.CompleteAsyncTask(ctx, w.db, taskID, finishedAt); err != nil { + logger.Warn(ctx, "Failed to mark task as completed", + "task_id", taskID, + "error", err, + ) + } + } else { + if err := database.FailAsyncTask(ctx, w.db, taskID, finishedAt); err != nil { + logger.Warn(ctx, "Failed to mark task as failed", + "task_id", taskID, + "error", err, + ) + } + } + } + + logger.Debug(ctx, "Task status updated", "task_id", taskID, "status", status, ) @@ -328,10 +417,24 @@ func (w *TaskWorker) updateTaskStatus(ctx context.Context, taskID uuid.UUID, sta // updateTaskWithError updates a task with error information func (w *TaskWorker) updateTaskWithError(ctx context.Context, taskID uuid.UUID, err error) error { - logger.Debug(ctx, "Updating task with error", + // Update task error information in database + errorMsg := err.Error() + stackTrace := fmt.Sprintf("%+v", err) + + updateErr := database.UpdateTaskErrorInfo(ctx, w.db, taskID, errorMsg, stackTrace) + if updateErr != nil { + logger.Error(ctx, "Failed to update task error info", + "task_id", taskID, + "error", updateErr, + ) + return updateErr + } + + logger.Warn(ctx, "Task failed with error", "task_id", taskID, - "error", err.Error(), + "error", errorMsg, ) + return nil } @@ -379,12 +482,16 @@ func (w *TaskWorker) checkHealth() { w.metrics.LastHealthCheck = time.Now() logger.Info(w.ctx, "Worker health check", - "tasks_processed", w.metrics.TasksProcessed, - "tasks_failed", w.metrics.TasksFailed, + "tasks_processed", w.metrics.TotalProcessed, + "tasks_failed", w.metrics.TotalFailed, + "tasks_success", w.metrics.TotalSuccess, "tasks_in_progress", w.metrics.TasksInProgress, "queue_depth", w.metrics.QueueDepth, + "queue_latency_ms", w.metrics.QueueLatency.Milliseconds(), "workers_active", w.metrics.WorkersActive, "workers_idle", w.metrics.WorkersIdle, + "memory_usage_mb", w.metrics.MemoryUsage/(1024*1024), + "cpu_load_percent", w.metrics.CPULoad, "pool_capacity", w.pool.Cap(), ) } @@ -418,14 +525,47 @@ func (w *TaskWorker) Stop() error { func (w *TaskWorker) GetMetrics() *WorkerMetrics { w.metrics.mu.RLock() defer w.metrics.mu.RUnlock() + + // Deep copy maps to avoid data races + tasksProcessedCopy := make(map[TaskType]int64) + for k, v := range w.metrics.TasksProcessed { + tasksProcessedCopy[k] = v + } + + tasksFailedCopy := make(map[TaskType]int64) + for k, v := range w.metrics.TasksFailed { + tasksFailedCopy[k] = v + } + + tasksSuccessCopy := make(map[TaskType]int64) + for k, v := range w.metrics.TasksSuccess { + tasksSuccessCopy[k] = v + } + + processingTimeCopy := make(map[TaskType]time.Duration) + for k, v := range w.metrics.ProcessingTime { + processingTimeCopy[k] = v + } + // Create a copy without the mutex to avoid copylocks warning return &WorkerMetrics{ - TasksProcessed: w.metrics.TasksProcessed, - TasksFailed: w.metrics.TasksFailed, + TasksProcessed: tasksProcessedCopy, + TasksFailed: tasksFailedCopy, + TasksSuccess: tasksSuccessCopy, + ProcessingTime: processingTimeCopy, + TotalProcessed: w.metrics.TotalProcessed, + TotalFailed: w.metrics.TotalFailed, + TotalSuccess: w.metrics.TotalSuccess, TasksInProgress: w.metrics.TasksInProgress, QueueDepth: w.metrics.QueueDepth, + QueueLatency: w.metrics.QueueLatency, WorkersActive: w.metrics.WorkersActive, WorkersIdle: w.metrics.WorkersIdle, + MemoryUsage: w.metrics.MemoryUsage, + CPULoad: w.metrics.CPULoad, + LastMinuteRate: w.metrics.LastMinuteRate, + Last5MinutesRate: w.metrics.Last5MinutesRate, + LastHourRate: w.metrics.LastHourRate, LastHealthCheck: w.metrics.LastHealthCheck, // Mutex is intentionally omitted } @@ -438,4 +578,34 @@ func (w *TaskWorker) IsHealthy() bool { // Consider unhealthy if last health check was too long ago return time.Since(w.metrics.LastHealthCheck) < 2*w.cfg.PollingInterval -} \ No newline at end of file +} +// RecordMetrics periodically records worker metrics to the logging system +func (w *TaskWorker) RecordMetrics(interval time.Duration) { + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + metricsLogger := NewMetricsLogger(w.ctx) + + for { + select { + case <-w.stopChan: + return + case <-ticker.C: + w.metrics.mu.RLock() + metricsLogger.LogWorkerMetrics( + w.metrics.WorkersActive, + w.metrics.WorkersIdle, + w.pool.Cap(), + w.metrics.MemoryUsage, + w.metrics.CPULoad, + ) + metricsLogger.LogQueueMetrics( + w.metrics.QueueDepth, + w.metrics.QueueLatency, + ) + w.metrics.mu.RUnlock() + } + } + }() +} From 4d5fcbc37638c9e5b44f91241638051a34907c8d Mon Sep 17 00:00:00 2001 From: douxu Date: Tue, 14 Apr 2026 17:00:30 +0800 Subject: [PATCH 27/43] Refactor async task system with unified task interfaces and add test task type - Create task/types_v2.go with unified task type definitions and interfaces * Add UnifiedTaskType and UnifiedTaskStatus constants * Define Task:Params interface for parameter validation and serialization * Define UnifiedTask interface as base for all task implementations * Add BaseTask for common task functionality --- handler/async_task_handler.go | 8 ++ orm/async_task.go | 4 +- task/handler_factory.go | 1 + task/test_task.go | 169 ++++++++++++++++++++++++++++++++++ task/types_v2.go | 138 +++++++++++++++++++++++++++ 5 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 task/test_task.go create mode 100644 task/types_v2.go diff --git a/handler/async_task_handler.go b/handler/async_task_handler.go index 2d7c3e9..a711177 100644 --- a/handler/async_task_handler.go +++ b/handler/async_task_handler.go @@ -395,6 +395,8 @@ func validateTaskParams(taskType string, params map[string]any) bool { return validateEventAnalysisParams(params) case string(orm.AsyncTaskTypeBatchImport): return validateBatchImportParams(params) + case string(orm.AsyncTaskTypeTest): + return validateTestTaskParams(params) default: return false } @@ -437,6 +439,12 @@ func validateBatchImportParams(params map[string]any) bool { return true } +func validateTestTaskParams(params map[string]any) bool { + // Test task has optional parameters, all are valid + // sleep_duration defaults to 60 seconds if not provided + return true +} + func splitCommaSeparated(s string) []string { var result []string var current strings.Builder diff --git a/orm/async_task.go b/orm/async_task.go index 40e312d..37709bf 100644 --- a/orm/async_task.go +++ b/orm/async_task.go @@ -17,6 +17,8 @@ const ( AsyncTaskTypeEventAnalysis AsyncTaskType = "EVENT_ANALYSIS" // AsyncTaskTypeBatchImport represents batch import task AsyncTaskTypeBatchImport AsyncTaskType = "BATCH_IMPORT" + // AsyncTaskTypeTest represents test task for system verification + AsyncTaskTypeTest AsyncTaskType = "TEST" ) // AsyncTaskStatus defines the status of asynchronous task @@ -119,7 +121,7 @@ func (a *AsyncTask) IsFailed() bool { func IsValidAsyncTaskType(taskType string) bool { switch AsyncTaskType(taskType) { case AsyncTaskTypeTopologyAnalysis, AsyncTaskTypePerformanceAnalysis, - AsyncTaskTypeEventAnalysis, AsyncTaskTypeBatchImport: + AsyncTaskTypeEventAnalysis, AsyncTaskTypeBatchImport, AsyncTaskTypeTest: return true default: return false diff --git a/task/handler_factory.go b/task/handler_factory.go index 9f6b7c2..c472fef 100644 --- a/task/handler_factory.go +++ b/task/handler_factory.go @@ -65,6 +65,7 @@ func (f *HandlerFactory) CreateDefaultHandlers() { f.RegisterHandler(TypeTopologyAnalysis, &TopologyAnalysisHandler{}) f.RegisterHandler(TypeEventAnalysis, &EventAnalysisHandler{}) f.RegisterHandler(TypeBatchImport, &BatchImportHandler{}) + f.RegisterHandler(TaskType(TaskTypeTest), NewTestTaskHandler()) } // BaseHandler provides common functionality for all task handlers diff --git a/task/test_task.go b/task/test_task.go new file mode 100644 index 0000000..2d60bd4 --- /dev/null +++ b/task/test_task.go @@ -0,0 +1,169 @@ +// Package task provides test task implementation for system verification +package task + +import ( + "context" + "fmt" + "time" + + "modelRT/database" + "modelRT/logger" + "modelRT/orm" + + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// TestTaskParams defines parameters for test task +type TestTaskParams struct { + // SleepDuration specifies how long the task should sleep (in seconds) + // Default is 60 seconds as per requirement + SleepDuration int `json:"sleep_duration"` + // Message is a custom message to include in the result + Message string `json:"message,omitempty"` +} + +// Validate checks if test task parameters are valid +func (p *TestTaskParams) Validate() error { + // Default to 60 seconds if not specified + if p.SleepDuration <= 0 { + p.SleepDuration = 60 + } + + // Validate max duration (max 1 hour) + if p.SleepDuration > 3600 { + return fmt.Errorf("sleep duration cannot exceed 3600 seconds (1 hour)") + } + + return nil +} + +// GetType returns the task type +func (p *TestTaskParams) GetType() UnifiedTaskType { + return TaskTypeTest +} + +// ToMap converts parameters to map for database storage +func (p *TestTaskParams) ToMap() map[string]interface{} { + return map[string]interface{}{ + "sleep_duration": p.SleepDuration, + "message": p.Message, + } +} + +// FromMap populates parameters from map (for database retrieval) +func (p *TestTaskParams) FromMap(params map[string]interface{}) error { + if v, ok := params["sleep_duration"]; ok { + if duration, isFloat := v.(float64); isFloat { + p.SleepDuration = int(duration) + } else if duration, isInt := v.(int); isInt { + p.SleepDuration = duration + } + } + + if v, ok := params["message"]; ok { + if msg, isString := v.(string); isString { + p.Message = msg + } + } + + return nil +} + +// TestTask implements a test task that sleeps for specified duration +// This task contains no I/O operations as per requirements +type TestTask struct { + *BaseTask +} + +// NewTestTask creates a new TestTask instance +func NewTestTask(params TestTaskParams) *TestTask { + return &TestTask{ + BaseTask: NewBaseTask(TaskTypeTest, ¶ms, "test_task"), + } +} + +// Execute performs the test task logic (sleep without I/O operations) +func (t *TestTask) Execute(ctx context.Context, taskID uuid.UUID, db *gorm.DB) error { + params, ok := t.GetParams().(*TestTaskParams) + if !ok { + return fmt.Errorf("invalid parameter type for TestTask") + } + + logger.Info(ctx, "Starting test task execution", + "task_id", taskID, + "sleep_duration_seconds", params.SleepDuration, + "message", params.Message, + ) + + // Sleep for the specified duration without any I/O operations + // This is pure CPU-time wait as per requirements + sleepDuration := time.Duration(params.SleepDuration) * time.Second + time.Sleep(sleepDuration) + + // Build result + result := map[string]interface{}{ + "status": "completed", + "sleep_duration": params.SleepDuration, + "message": params.Message, + "executed_at": time.Now().Unix(), + "task_id": taskID.String(), + } + + // Save result to database + if err := database.UpdateAsyncTaskResultWithSuccess(ctx, db, taskID, orm.JSONMap(result)); err != nil { + logger.Error(ctx, "Failed to save test task result", + "task_id", taskID, + "error", err, + ) + return fmt.Errorf("failed to save task result: %w", err) + } + + logger.Info(ctx, "Test task completed successfully", + "task_id", taskID, + "sleep_duration_seconds", params.SleepDuration, + ) + + return nil +} + +// TestTaskHandler handles test task execution +type TestTaskHandler struct { + *BaseHandler +} + +// NewTestTaskHandler creates a new TestTaskHandler +func NewTestTaskHandler() *TestTaskHandler { + return &TestTaskHandler{ + BaseHandler: NewBaseHandler("test_task_handler"), + } +} + +// Execute processes a test task using the unified task interface +func (h *TestTaskHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, db *gorm.DB) error { + logger.Info(ctx, "Executing test task", + "task_id", taskID, + "task_type", taskType, + ) + + // Fetch task parameters from database + asyncTask, err := database.GetAsyncTaskByID(ctx, db, taskID) + if err != nil { + return fmt.Errorf("failed to fetch task: %w", err) + } + + // Convert params map to TestTaskParams + params := &TestTaskParams{} + if err := params.FromMap(map[string]interface{}(asyncTask.Params)); err != nil { + return fmt.Errorf("failed to parse task params: %w", err) + } + + // Create and execute test task + testTask := NewTestTask(*params) + return testTask.Execute(ctx, taskID, db) +} + +// CanHandle returns true for test tasks +func (h *TestTaskHandler) CanHandle(taskType TaskType) bool { + return string(TaskTypeTest) == string(taskType) +} diff --git a/task/types_v2.go b/task/types_v2.go new file mode 100644 index 0000000..aed3558 --- /dev/null +++ b/task/types_v2.go @@ -0,0 +1,138 @@ +// Package task provides unified task type definitions and interfaces +package task + +import ( + "context" + "fmt" + + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// UnifiedTaskType defines all async task types in a single location +type UnifiedTaskType string + +const ( + // TaskTypeTopologyAnalysis represents topology analysis task + TaskTypeTopologyAnalysis UnifiedTaskType = "TOPOLOGY_ANALYSIS" + // TaskTypePerformanceAnalysis represents performance analysis task + TaskTypePerformanceAnalysis UnifiedTaskType = "PERFORMANCE_ANALYSIS" + // TaskTypeEventAnalysis represents event analysis task + TaskTypeEventAnalysis UnifiedTaskType = "EVENT_ANALYSIS" + // TaskTypeBatchImport represents batch import task + TaskTypeBatchImport UnifiedTaskType = "BATCH_IMPORT" + // TaskTypeTest represents test task for system verification + TaskTypeTest UnifiedTaskType = "TEST" +) + +// UnifiedTaskStatus defines task status constants +type UnifiedTaskStatus string + +const ( + // TaskStatusPending represents task waiting to be processed + TaskStatusPending UnifiedTaskStatus = "PENDING" + // TaskStatusRunning represents task currently executing + TaskStatusRunning UnifiedTaskStatus = "RUNNING" + // TaskStatusCompleted represents task finished successfully + TaskStatusCompleted UnifiedTaskStatus = "COMPLETED" + // TaskStatusFailed represents task failed with error + TaskStatusFailed UnifiedTaskStatus = "FAILED" +) + +// TaskParams defines the interface for task-specific parameters +// All task types must implement this interface to provide their parameter structure +type TaskParams interface { + // Validate checks if the parameters are valid for this task type + Validate() error + // GetType returns the task type these parameters are for + GetType() UnifiedTaskType + // ToMap converts parameters to map for database storage + ToMap() map[string]interface{} + // FromMap populates parameters from map (for database retrieval) + FromMap(params map[string]interface{}) error +} + +// UnifiedTask defines the base interface that all tasks must implement +// This provides a clean abstraction for task execution and management +type UnifiedTask interface { + // GetType returns the task type + GetType() UnifiedTaskType + + // GetParams returns the task parameters + GetParams() TaskParams + + // Execute performs the actual task logic + Execute(ctx context.Context, taskID uuid.UUID, db *gorm.DB) error + + // GetName returns a human-readable task name for logging + GetName() string + + // Validate checks if the task is valid before execution + Validate() error +} + +// BaseTask provides common functionality for all task implementations +type BaseTask struct { + taskType UnifiedTaskType + params TaskParams + name string +} + +// NewBaseTask creates a new BaseTask instance +func NewBaseTask(taskType UnifiedTaskType, params TaskParams, name string) *BaseTask { + return &BaseTask{ + taskType: taskType, + params: params, + name: name, + } +} + +// GetType returns the task type +func (t *BaseTask) GetType() UnifiedTaskType { + return t.taskType +} + +// GetParams returns the task parameters +func (t *BaseTask) GetParams() TaskParams { + return t.params +} + +// GetName returns the task name +func (t *BaseTask) GetName() string { + return t.name +} + +// Validate checks if the task is valid +func (t *BaseTask) Validate() error { + if t.params == nil { + return fmt.Errorf("task parameters cannot be nil") + } + + if t.taskType != t.params.GetType() { + return fmt.Errorf("task type mismatch: expected %s, got %s", t.taskType, t.params.GetType()) + } + + return t.params.Validate() +} + +// IsTaskType checks if a task type string is valid +func IsTaskType(taskType string) bool { + switch UnifiedTaskType(taskType) { + case TaskTypeTopologyAnalysis, TaskTypePerformanceAnalysis, + TaskTypeEventAnalysis, TaskTypeBatchImport, TaskTypeTest: + return true + default: + return false + } +} + +// GetTaskTypes returns all registered task types +func GetTaskTypes() []UnifiedTaskType { + return []UnifiedTaskType{ + TaskTypeTopologyAnalysis, + TaskTypePerformanceAnalysis, + TaskTypeEventAnalysis, + TaskTypeBatchImport, + TaskTypeTest, + } +} From 4a3f7a65bc2b46846341c91b40ec40a41e76c258 Mon Sep 17 00:00:00 2001 From: douxu Date: Fri, 17 Apr 2026 14:09:02 +0800 Subject: [PATCH 28/43] Refactor async task handlers into specialized handlers Split monolithic async_task_handler.go into separate handlers: - async_task_cancel_handler.go: Handles task cancellation - async_task_create_handler.go: Handles task creation - async_task_progress_update_handler.go: Handles progress updates - async_task_result_detail_handler.go: Handles result details - async_task_result_query_handler.go: Handles result queries - async_task_status_update_handler.go: Handles status updates This improves code organization and maintainability by separating concerns. Co-Authored-By: Claude Opus 4.6 --- handler/async_task_cancel_handler.go | 119 +++ handler/async_task_create_handler.go | 242 ++++++ handler/async_task_handler.go | 713 ------------------ handler/async_task_progress_update_handler.go | 54 ++ handler/async_task_result_detail_handler.go | 124 +++ handler/async_task_result_query_handler.go | 144 ++++ handler/async_task_status_update_handler.go | 89 +++ 7 files changed, 772 insertions(+), 713 deletions(-) create mode 100644 handler/async_task_cancel_handler.go create mode 100644 handler/async_task_create_handler.go delete mode 100644 handler/async_task_handler.go create mode 100644 handler/async_task_progress_update_handler.go create mode 100644 handler/async_task_result_detail_handler.go create mode 100644 handler/async_task_result_query_handler.go create mode 100644 handler/async_task_status_update_handler.go diff --git a/handler/async_task_cancel_handler.go b/handler/async_task_cancel_handler.go new file mode 100644 index 0000000..9b99b52 --- /dev/null +++ b/handler/async_task_cancel_handler.go @@ -0,0 +1,119 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "net/http" + "time" + + "modelRT/database" + "modelRT/logger" + "modelRT/network" + "modelRT/orm" + + "github.com/gin-gonic/gin" + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// AsyncTaskCancelHandler handles cancellation of an async task +// @Summary 取消异步任务 +// @Description 取消指定ID的异步任务（如果任务尚未开始执行） +// @Tags AsyncTask +// @Accept json +// @Produce json +// @Param task_id path string true "任务ID" +// @Success 200 {object} network.SuccessResponse "任务取消成功" +// @Failure 400 {object} network.FailureResponse "请求参数错误或任务无法取消" +// @Failure 404 {object} network.FailureResponse "任务不存在" +// @Failure 500 {object} network.FailureResponse "服务器内部错误" +// @Router /task/async/{task_id}/cancel [post] +func AsyncTaskCancelHandler(c *gin.Context) { + ctx := c.Request.Context() + + // Parse task ID from path parameter + taskIDStr := c.Param("task_id") + if taskIDStr == "" { + logger.Error(ctx, "task_id parameter is required") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "task_id parameter is required", + }) + return + } + + taskID, err := uuid.FromString(taskIDStr) + if err != nil { + logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid task ID format", + }) + return + } + + pgClient := database.GetPostgresDBClient() + if pgClient == nil { + logger.Error(ctx, "database connection not found in context") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "database connection error", + }) + return + } + + // Query task from database + asyncTask, err := database.GetAsyncTaskByID(ctx, pgClient, taskID) + if err != nil { + if err == gorm.ErrRecordNotFound { + logger.Error(ctx, "async task not found", "task_id", taskID) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusNotFound, + Msg: "task not found", + }) + return + } + logger.Error(ctx, "failed to query async task from database", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to query task", + }) + return + } + + // Check if task can be cancelled (only SUBMITTED tasks can be cancelled) + if asyncTask.Status != orm.AsyncTaskStatusSubmitted { + logger.Error(ctx, "task cannot be cancelled", "task_id", taskID, "status", asyncTask.Status) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "task cannot be cancelled (already running or completed)", + }) + return + } + + // Update task status to failed with cancellation reason + timestamp := time.Now().Unix() + err = database.FailAsyncTask(ctx, pgClient, taskID, timestamp) + if err != nil { + logger.Error(ctx, "failed to cancel async task", "task_id", taskID, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to cancel task", + }) + return + } + + // Update task result with cancellation error + err = database.UpdateAsyncTaskResultWithError(ctx, pgClient, taskID, 40003, "task cancelled by user", orm.JSONMap{ + "cancelled_at": timestamp, + "cancelled_by": "user", + }) + if err != nil { + logger.Error(ctx, "failed to update task result with cancellation error", "task_id", taskID, "error", err) + // Continue anyway since task is already marked as failed + } + + c.JSON(http.StatusOK, network.SuccessResponse{ + Code: 2000, + Msg: "task cancelled successfully", + }) +} diff --git a/handler/async_task_create_handler.go b/handler/async_task_create_handler.go new file mode 100644 index 0000000..630b594 --- /dev/null +++ b/handler/async_task_create_handler.go @@ -0,0 +1,242 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "net/http" + "strings" + + "modelRT/config" + "modelRT/database" + "modelRT/logger" + "modelRT/network" + "modelRT/orm" + "modelRT/task" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" +) + +// AsyncTaskCreateHandler handles creation of asynchronous tasks +// @Summary 创建异步任务 +// @Description 创建新的异步任务并返回任务ID，任务将被提交到队列等待处理 +// @Tags AsyncTask +// @Accept json +// @Produce json +// @Param request body network.AsyncTaskCreateRequest true "任务创建请求" +// @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskCreateResponse} "任务创建成功" +// @Failure 400 {object} network.FailureResponse "请求参数错误" +// @Failure 500 {object} network.FailureResponse "服务器内部错误" +// @Router /task/async [post] +func AsyncTaskCreateHandler(c *gin.Context) { + ctx := c.Request.Context() + var request network.AsyncTaskCreateRequest + + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "failed to unmarshal async task create request", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid request parameters", + }) + return + } + + // Validate task type + if !orm.IsValidAsyncTaskType(request.TaskType) { + logger.Error(ctx, "invalid task type", "task_type", request.TaskType) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid task type", + }) + return + } + + // Validate task parameters based on task type + if !validateTaskParams(request.TaskType, request.Params) { + logger.Error(ctx, "invalid task parameters", "task_type", request.TaskType, "params", request.Params) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid task parameters", + }) + return + } + + pgClient := database.GetPostgresDBClient() + if pgClient == nil { + logger.Error(ctx, "database connection not found in context") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "database connection error", + }) + return + } + + // Create task in database + taskType := orm.AsyncTaskType(request.TaskType) + params := orm.JSONMap(request.Params) + + asyncTask, err := database.CreateAsyncTask(ctx, pgClient, taskType, params) + if err != nil { + logger.Error(ctx, "failed to create async task in database", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to create task", + }) + return + } + + // Send task to message queue + cfg, exists := c.Get("config") + if !exists { + logger.Warn(ctx, "Configuration not found in context, skipping queue publishing") + } else { + modelRTConfig := cfg.(config.ModelRTConfig) + ctx := c.Request.Context() + + // Create queue producer + producer, err := task.NewQueueProducer(ctx, modelRTConfig.RabbitMQConfig) + if err != nil { + logger.Error(ctx, "Failed to create queue producer", "error", err) + // Continue without queue publishing + } else { + defer producer.Close() + + // Publish task to queue + taskType := task.TaskType(request.TaskType) + priority := 5 // Default priority + + if err := producer.PublishTaskWithRetry(ctx, asyncTask.TaskID, taskType, priority, 3); err != nil { + logger.Error(ctx, "Failed to publish task to queue", + "task_id", asyncTask.TaskID, + "error", err) + // Log error but don't affect task creation response + } else { + logger.Info(ctx, "Task published to queue successfully", + "task_id", asyncTask.TaskID, + "queue", task.TaskQueueName) + } + } + } + + logger.Info(ctx, "async task created successfully", "task_id", asyncTask.TaskID, "task_type", request.TaskType) + + // Return success response + c.JSON(http.StatusOK, network.SuccessResponse{ + Code: 2000, + Msg: "task created successfully", + Payload: network.AsyncTaskCreateResponse{ + TaskID: asyncTask.TaskID, + }, + }) +} + +func validateTaskParams(taskType string, params map[string]any) bool { + switch taskType { + case string(orm.AsyncTaskTypeTopologyAnalysis): + return validateTopologyAnalysisParams(params) + case string(orm.AsyncTaskTypePerformanceAnalysis): + return validatePerformanceAnalysisParams(params) + case string(orm.AsyncTaskTypeEventAnalysis): + return validateEventAnalysisParams(params) + case string(orm.AsyncTaskTypeBatchImport): + return validateBatchImportParams(params) + case string(orm.AsyncTaskTypeTest): + return validateTestTaskParams(params) + default: + return false + } +} + +func validateTopologyAnalysisParams(params map[string]any) bool { + // Check required parameters for topology analysis + if startUUID, ok := params["start_uuid"]; !ok || startUUID == "" { + return false + } + if endUUID, ok := params["end_uuid"]; !ok || endUUID == "" { + return false + } + return true +} + +func validatePerformanceAnalysisParams(params map[string]any) bool { + // Check required parameters for performance analysis + if componentIDs, ok := params["component_ids"]; !ok { + return false + } else if ids, isSlice := componentIDs.([]interface{}); !isSlice || len(ids) == 0 { + return false + } + return true +} + +func validateEventAnalysisParams(params map[string]any) bool { + // Check required parameters for event analysis + if eventType, ok := params["event_type"]; !ok || eventType == "" { + return false + } + return true +} + +func validateBatchImportParams(params map[string]any) bool { + // Check required parameters for batch import + if filePath, ok := params["file_path"]; !ok || filePath == "" { + return false + } + return true +} + +func validateTestTaskParams(params map[string]any) bool { + // Test task has optional parameters, all are valid + // sleep_duration defaults to 60 seconds if not provided + return true +} + +func splitCommaSeparated(s string) []string { + var result []string + var current strings.Builder + inQuotes := false + escape := false + + for _, ch := range s { + if escape { + current.WriteRune(ch) + escape = false + continue + } + + switch ch { + case '\\': + escape = true + case '"': + inQuotes = !inQuotes + case ',': + if !inQuotes { + result = append(result, strings.TrimSpace(current.String())) + current.Reset() + } else { + current.WriteRune(ch) + } + default: + current.WriteRune(ch) + } + } + + if current.Len() > 0 { + result = append(result, strings.TrimSpace(current.String())) + } + + return result +} + +func getDBFromContext(c *gin.Context) *gorm.DB { + // Try to get database connection from context + // This should be set by middleware + if db, exists := c.Get("db"); exists { + if gormDB, ok := db.(*gorm.DB); ok { + return gormDB + } + } + + // Fallback to global database connection + // This should be implemented based on your application's database setup + // For now, return nil - actual implementation should retrieve from application context + return nil +} diff --git a/handler/async_task_handler.go b/handler/async_task_handler.go deleted file mode 100644 index a711177..0000000 --- a/handler/async_task_handler.go +++ /dev/null @@ -1,713 +0,0 @@ -// Package handler provides HTTP handlers for various endpoints. -package handler - -import ( - "net/http" - "strings" - "time" - - "modelRT/config" - "modelRT/database" - "modelRT/logger" - "modelRT/network" - "modelRT/orm" - "modelRT/task" - - "github.com/gin-gonic/gin" - "github.com/gofrs/uuid" - "gorm.io/gorm" -) - -// AsyncTaskCreateHandler handles creation of asynchronous tasks -// @Summary 创建异步任务 -// @Description 创建新的异步任务并返回任务ID，任务将被提交到队列等待处理 -// @Tags AsyncTask -// @Accept json -// @Produce json -// @Param request body network.AsyncTaskCreateRequest true "任务创建请求" -// @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskCreateResponse} "任务创建成功" -// @Failure 400 {object} network.FailureResponse "请求参数错误" -// @Failure 500 {object} network.FailureResponse "服务器内部错误" -// @Router /task/async [post] -func AsyncTaskCreateHandler(c *gin.Context) { - ctx := c.Request.Context() - var request network.AsyncTaskCreateRequest - - if err := c.ShouldBindJSON(&request); err != nil { - logger.Error(ctx, "failed to unmarshal async task create request", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid request parameters", - }) - return - } - - // Validate task type - if !orm.IsValidAsyncTaskType(request.TaskType) { - logger.Error(ctx, "invalid task type", "task_type", request.TaskType) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid task type", - }) - return - } - - // Validate task parameters based on task type - if !validateTaskParams(request.TaskType, request.Params) { - logger.Error(ctx, "invalid task parameters", "task_type", request.TaskType, "params", request.Params) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid task parameters", - }) - return - } - - // Get database connection from context or use default - db := getDBFromContext(c) - if db == nil { - logger.Error(ctx, "database connection not found in context") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "database connection error", - }) - return - } - - // Create task in database - taskType := orm.AsyncTaskType(request.TaskType) - params := orm.JSONMap(request.Params) - - asyncTask, err := database.CreateAsyncTask(ctx, db, taskType, params) - if err != nil { - logger.Error(ctx, "failed to create async task in database", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to create task", - }) - return - } - - // Send task to message queue - cfg, exists := c.Get("config") - if !exists { - logger.Warn(ctx, "Configuration not found in context, skipping queue publishing") - } else { - modelRTConfig := cfg.(config.ModelRTConfig) - ctx := c.Request.Context() - - // Create queue producer - producer, err := task.NewQueueProducer(ctx, modelRTConfig.RabbitMQConfig) - if err != nil { - logger.Error(ctx, "Failed to create queue producer", "error", err) - // Continue without queue publishing - } else { - defer producer.Close() - - // Publish task to queue - taskType := task.TaskType(request.TaskType) - priority := 5 // Default priority - - if err := producer.PublishTaskWithRetry(ctx, asyncTask.TaskID, taskType, priority, 3); err != nil { - logger.Error(ctx, "Failed to publish task to queue", - "task_id", asyncTask.TaskID, - "error", err) - // Log error but don't affect task creation response - } else { - logger.Info(ctx, "Task published to queue successfully", - "task_id", asyncTask.TaskID, - "queue", task.TaskQueueName) - } - } - } - - logger.Info(ctx, "async task created successfully", "task_id", asyncTask.TaskID, "task_type", request.TaskType) - - // Return success response - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: 2000, - Msg: "task created successfully", - Payload: network.AsyncTaskCreateResponse{ - TaskID: asyncTask.TaskID, - }, - }) -} - -// AsyncTaskResultQueryHandler handles querying of asynchronous task results -// @Summary 查询异步任务结果 -// @Description 根据任务ID列表查询异步任务的状态和结果 -// @Tags AsyncTask -// @Accept json -// @Produce json -// @Param task_ids query string true "任务ID列表，用逗号分隔" -// @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskResultQueryResponse} "查询成功" -// @Failure 400 {object} network.FailureResponse "请求参数错误" -// @Failure 500 {object} network.FailureResponse "服务器内部错误" -// @Router /task/async/results [get] -func AsyncTaskResultQueryHandler(c *gin.Context) { - ctx := c.Request.Context() - - // Parse task IDs from query parameter - taskIDsParam := c.Query("task_ids") - if taskIDsParam == "" { - logger.Error(ctx, "task_ids parameter is required") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "task_ids parameter is required", - }) - return - } - - // Parse comma-separated task IDs - var taskIDs []uuid.UUID - taskIDStrs := splitCommaSeparated(taskIDsParam) - for _, taskIDStr := range taskIDStrs { - taskID, err := uuid.FromString(taskIDStr) - if err != nil { - logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid task ID format", - }) - return - } - taskIDs = append(taskIDs, taskID) - } - - if len(taskIDs) == 0 { - logger.Error(ctx, "no valid task IDs provided") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "no valid task IDs provided", - }) - return - } - - // Get database connection from context or use default - db := getDBFromContext(c) - if db == nil { - logger.Error(ctx, "database connection not found in context") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "database connection error", - }) - return - } - - // Query tasks from database - asyncTasks, err := database.GetAsyncTasksByIDs(ctx, db, taskIDs) - if err != nil { - logger.Error(ctx, "failed to query async tasks from database", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to query tasks", - }) - return - } - - // Query task results from database - taskResults, err := database.GetAsyncTaskResults(ctx, db, taskIDs) - if err != nil { - logger.Error(ctx, "failed to query async task results from database", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to query task results", - }) - return - } - - // Create a map of task results for easy lookup - taskResultMap := make(map[uuid.UUID]orm.AsyncTaskResult) - for _, result := range taskResults { - taskResultMap[result.TaskID] = result - } - - // Convert to response format - var responseTasks []network.AsyncTaskResult - for _, asyncTask := range asyncTasks { - taskResult := network.AsyncTaskResult{ - TaskID: asyncTask.TaskID, - TaskType: string(asyncTask.TaskType), - Status: string(asyncTask.Status), - CreatedAt: asyncTask.CreatedAt, - FinishedAt: asyncTask.FinishedAt, - Progress: asyncTask.Progress, - } - - // Add result or error information if available - if result, exists := taskResultMap[asyncTask.TaskID]; exists { - if result.Result != nil { - taskResult.Result = map[string]any(result.Result) - } - if result.ErrorCode != nil { - taskResult.ErrorCode = result.ErrorCode - } - if result.ErrorMessage != nil { - taskResult.ErrorMessage = result.ErrorMessage - } - if result.ErrorDetail != nil { - taskResult.ErrorDetail = map[string]any(result.ErrorDetail) - } - } - - responseTasks = append(responseTasks, taskResult) - } - - // Return success response - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: 2000, - Msg: "query completed", - Payload: network.AsyncTaskResultQueryResponse{ - Total: len(responseTasks), - Tasks: responseTasks, - }, - }) -} - -// AsyncTaskProgressUpdateHandler handles updating task progress (internal use, not exposed via API) -func AsyncTaskProgressUpdateHandler(c *gin.Context) { - ctx := c.Request.Context() - var request network.AsyncTaskProgressUpdate - - if err := c.ShouldBindJSON(&request); err != nil { - logger.Error(ctx, "failed to unmarshal async task progress update request", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid request parameters", - }) - return - } - - // Get database connection from context or use default - db := getDBFromContext(c) - if db == nil { - logger.Error(ctx, "database connection not found in context") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "database connection error", - }) - return - } - - // Update task progress - err := database.UpdateAsyncTaskProgress(ctx, db, request.TaskID, request.Progress) - if err != nil { - logger.Error(ctx, "failed to update async task progress", "task_id", request.TaskID, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to update task progress", - }) - return - } - - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: 2000, - Msg: "task progress updated successfully", - Payload: nil, - }) -} - -// AsyncTaskStatusUpdateHandler handles updating task status (internal use, not exposed via API) -func AsyncTaskStatusUpdateHandler(c *gin.Context) { - ctx := c.Request.Context() - var request network.AsyncTaskStatusUpdate - - if err := c.ShouldBindJSON(&request); err != nil { - logger.Error(ctx, "failed to unmarshal async task status update request", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid request parameters", - }) - return - } - - // Validate status - validStatus := map[string]bool{ - string(orm.AsyncTaskStatusSubmitted): true, - string(orm.AsyncTaskStatusRunning): true, - string(orm.AsyncTaskStatusCompleted): true, - string(orm.AsyncTaskStatusFailed): true, - } - - if !validStatus[request.Status] { - logger.Error(ctx, "invalid task status", "status", request.Status) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid task status", - }) - return - } - - // Get database connection from context or use default - db := getDBFromContext(c) - if db == nil { - logger.Error(ctx, "database connection not found in context") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "database connection error", - }) - return - } - - // Update task status - status := orm.AsyncTaskStatus(request.Status) - err := database.UpdateAsyncTaskStatus(ctx, db, request.TaskID, status) - if err != nil { - logger.Error(ctx, "failed to update async task status", "task_id", request.TaskID, "status", request.Status, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to update task status", - }) - return - } - - // If task is completed or failed, update finished_at timestamp - if request.Status == string(orm.AsyncTaskStatusCompleted) { - err = database.CompleteAsyncTask(ctx, db, request.TaskID, request.Timestamp) - } else if request.Status == string(orm.AsyncTaskStatusFailed) { - err = database.FailAsyncTask(ctx, db, request.TaskID, request.Timestamp) - } - - if err != nil { - logger.Error(ctx, "failed to update async task completion timestamp", "task_id", request.TaskID, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to update task completion timestamp", - }) - return - } - - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: 2000, - Msg: "task status updated successfully", - Payload: nil, - }) -} - -// Helper functions - -func validateTaskParams(taskType string, params map[string]any) bool { - switch taskType { - case string(orm.AsyncTaskTypeTopologyAnalysis): - return validateTopologyAnalysisParams(params) - case string(orm.AsyncTaskTypePerformanceAnalysis): - return validatePerformanceAnalysisParams(params) - case string(orm.AsyncTaskTypeEventAnalysis): - return validateEventAnalysisParams(params) - case string(orm.AsyncTaskTypeBatchImport): - return validateBatchImportParams(params) - case string(orm.AsyncTaskTypeTest): - return validateTestTaskParams(params) - default: - return false - } -} - -func validateTopologyAnalysisParams(params map[string]any) bool { - // Check required parameters for topology analysis - if startUUID, ok := params["start_uuid"]; !ok || startUUID == "" { - return false - } - if endUUID, ok := params["end_uuid"]; !ok || endUUID == "" { - return false - } - return true -} - -func validatePerformanceAnalysisParams(params map[string]any) bool { - // Check required parameters for performance analysis - if componentIDs, ok := params["component_ids"]; !ok { - return false - } else if ids, isSlice := componentIDs.([]interface{}); !isSlice || len(ids) == 0 { - return false - } - return true -} - -func validateEventAnalysisParams(params map[string]any) bool { - // Check required parameters for event analysis - if eventType, ok := params["event_type"]; !ok || eventType == "" { - return false - } - return true -} - -func validateBatchImportParams(params map[string]any) bool { - // Check required parameters for batch import - if filePath, ok := params["file_path"]; !ok || filePath == "" { - return false - } - return true -} - -func validateTestTaskParams(params map[string]any) bool { - // Test task has optional parameters, all are valid - // sleep_duration defaults to 60 seconds if not provided - return true -} - -func splitCommaSeparated(s string) []string { - var result []string - var current strings.Builder - inQuotes := false - escape := false - - for _, ch := range s { - if escape { - current.WriteRune(ch) - escape = false - continue - } - - switch ch { - case '\\': - escape = true - case '"': - inQuotes = !inQuotes - case ',': - if !inQuotes { - result = append(result, strings.TrimSpace(current.String())) - current.Reset() - } else { - current.WriteRune(ch) - } - default: - current.WriteRune(ch) - } - } - - if current.Len() > 0 { - result = append(result, strings.TrimSpace(current.String())) - } - - return result -} - -func getDBFromContext(c *gin.Context) *gorm.DB { - // Try to get database connection from context - // This should be set by middleware - if db, exists := c.Get("db"); exists { - if gormDB, ok := db.(*gorm.DB); ok { - return gormDB - } - } - - // Fallback to global database connection - // This should be implemented based on your application's database setup - // For now, return nil - actual implementation should retrieve from application context - return nil -} - -// AsyncTaskResultDetailHandler handles detailed query of a single async task result -// @Summary 查询异步任务详情 -// @Description 根据任务ID查询异步任务的详细状态和结果 -// @Tags AsyncTask -// @Accept json -// @Produce json -// @Param task_id path string true "任务ID" -// @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskResult} "查询成功" -// @Failure 400 {object} network.FailureResponse "请求参数错误" -// @Failure 404 {object} network.FailureResponse "任务不存在" -// @Failure 500 {object} network.FailureResponse "服务器内部错误" -// @Router /task/async/{task_id} [get] -func AsyncTaskResultDetailHandler(c *gin.Context) { - ctx := c.Request.Context() - - // Parse task ID from path parameter - taskIDStr := c.Param("task_id") - if taskIDStr == "" { - logger.Error(ctx, "task_id parameter is required") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "task_id parameter is required", - }) - return - } - - taskID, err := uuid.FromString(taskIDStr) - if err != nil { - logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid task ID format", - }) - return - } - - // Get database connection from context or use default - db := getDBFromContext(c) - if db == nil { - logger.Error(ctx, "database connection not found in context") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "database connection error", - }) - return - } - - // Query task from database - asyncTask, err := database.GetAsyncTaskByID(ctx, db, taskID) - if err != nil { - if err == gorm.ErrRecordNotFound { - logger.Error(ctx, "async task not found", "task_id", taskID) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusNotFound, - Msg: "task not found", - }) - return - } - logger.Error(ctx, "failed to query async task from database", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to query task", - }) - return - } - - // Query task result from database - taskResult, err := database.GetAsyncTaskResult(ctx, db, taskID) - if err != nil { - logger.Error(ctx, "failed to query async task result from database", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to query task result", - }) - return - } - - // Convert to response format - responseTask := network.AsyncTaskResult{ - TaskID: asyncTask.TaskID, - TaskType: string(asyncTask.TaskType), - Status: string(asyncTask.Status), - CreatedAt: asyncTask.CreatedAt, - FinishedAt: asyncTask.FinishedAt, - Progress: asyncTask.Progress, - } - - // Add result or error information if available - if taskResult != nil { - if taskResult.Result != nil { - responseTask.Result = map[string]any(taskResult.Result) - } - if taskResult.ErrorCode != nil { - responseTask.ErrorCode = taskResult.ErrorCode - } - if taskResult.ErrorMessage != nil { - responseTask.ErrorMessage = taskResult.ErrorMessage - } - if taskResult.ErrorDetail != nil { - responseTask.ErrorDetail = map[string]any(taskResult.ErrorDetail) - } - } - - // Return success response - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: 2000, - Msg: "query completed", - Payload: responseTask, - }) -} - -// AsyncTaskCancelHandler handles cancellation of an async task -// @Summary 取消异步任务 -// @Description 取消指定ID的异步任务（如果任务尚未开始执行） -// @Tags AsyncTask -// @Accept json -// @Produce json -// @Param task_id path string true "任务ID" -// @Success 200 {object} network.SuccessResponse "任务取消成功" -// @Failure 400 {object} network.FailureResponse "请求参数错误或任务无法取消" -// @Failure 404 {object} network.FailureResponse "任务不存在" -// @Failure 500 {object} network.FailureResponse "服务器内部错误" -// @Router /task/async/{task_id}/cancel [post] -func AsyncTaskCancelHandler(c *gin.Context) { - ctx := c.Request.Context() - - // Parse task ID from path parameter - taskIDStr := c.Param("task_id") - if taskIDStr == "" { - logger.Error(ctx, "task_id parameter is required") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "task_id parameter is required", - }) - return - } - - taskID, err := uuid.FromString(taskIDStr) - if err != nil { - logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid task ID format", - }) - return - } - - // Get database connection from context or use default - db := getDBFromContext(c) - if db == nil { - logger.Error(ctx, "database connection not found in context") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "database connection error", - }) - return - } - - // Query task from database - asyncTask, err := database.GetAsyncTaskByID(ctx, db, taskID) - if err != nil { - if err == gorm.ErrRecordNotFound { - logger.Error(ctx, "async task not found", "task_id", taskID) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusNotFound, - Msg: "task not found", - }) - return - } - logger.Error(ctx, "failed to query async task from database", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to query task", - }) - return - } - - // Check if task can be cancelled (only SUBMITTED tasks can be cancelled) - if asyncTask.Status != orm.AsyncTaskStatusSubmitted { - logger.Error(ctx, "task cannot be cancelled", "task_id", taskID, "status", asyncTask.Status) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "task cannot be cancelled (already running or completed)", - }) - return - } - - // Update task status to failed with cancellation reason - timestamp := time.Now().Unix() - err = database.FailAsyncTask(ctx, db, taskID, timestamp) - if err != nil { - logger.Error(ctx, "failed to cancel async task", "task_id", taskID, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to cancel task", - }) - return - } - - // Update task result with cancellation error - err = database.UpdateAsyncTaskResultWithError(ctx, db, taskID, 40003, "task cancelled by user", orm.JSONMap{ - "cancelled_at": timestamp, - "cancelled_by": "user", - }) - if err != nil { - logger.Error(ctx, "failed to update task result with cancellation error", "task_id", taskID, "error", err) - // Continue anyway since task is already marked as failed - } - - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: 2000, - Msg: "task cancelled successfully", - }) -} diff --git a/handler/async_task_progress_update_handler.go b/handler/async_task_progress_update_handler.go new file mode 100644 index 0000000..ad3fe9b --- /dev/null +++ b/handler/async_task_progress_update_handler.go @@ -0,0 +1,54 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "net/http" + + "modelRT/database" + "modelRT/logger" + "modelRT/network" + + "github.com/gin-gonic/gin" +) + +// AsyncTaskProgressUpdateHandler handles updating task progress (internal use, not exposed via API) +func AsyncTaskProgressUpdateHandler(c *gin.Context) { + ctx := c.Request.Context() + var request network.AsyncTaskProgressUpdate + + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "failed to unmarshal async task progress update request", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid request parameters", + }) + return + } + + pgClient := database.GetPostgresDBClient() + if pgClient == nil { + logger.Error(ctx, "database connection not found in context") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "database connection error", + }) + return + } + + // Update task progress + err := database.UpdateAsyncTaskProgress(ctx, pgClient, request.TaskID, request.Progress) + if err != nil { + logger.Error(ctx, "failed to update async task progress", "task_id", request.TaskID, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to update task progress", + }) + return + } + + c.JSON(http.StatusOK, network.SuccessResponse{ + Code: 2000, + Msg: "task progress updated successfully", + Payload: nil, + }) +} diff --git a/handler/async_task_result_detail_handler.go b/handler/async_task_result_detail_handler.go new file mode 100644 index 0000000..1494945 --- /dev/null +++ b/handler/async_task_result_detail_handler.go @@ -0,0 +1,124 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "net/http" + + "modelRT/database" + "modelRT/logger" + "modelRT/network" + + "github.com/gin-gonic/gin" + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// AsyncTaskResultDetailHandler handles detailed query of a single async task result +// @Summary 查询异步任务详情 +// @Description 根据任务ID查询异步任务的详细状态和结果 +// @Tags AsyncTask +// @Accept json +// @Produce json +// @Param task_id path string true "任务ID" +// @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskResult} "查询成功" +// @Failure 400 {object} network.FailureResponse "请求参数错误" +// @Failure 404 {object} network.FailureResponse "任务不存在" +// @Failure 500 {object} network.FailureResponse "服务器内部错误" +// @Router /task/async/{task_id} [get] +func AsyncTaskResultDetailHandler(c *gin.Context) { + ctx := c.Request.Context() + + // Parse task ID from path parameter + taskIDStr := c.Param("task_id") + if taskIDStr == "" { + logger.Error(ctx, "task_id parameter is required") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "task_id parameter is required", + }) + return + } + + taskID, err := uuid.FromString(taskIDStr) + if err != nil { + logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid task ID format", + }) + return + } + + pgClient := database.GetPostgresDBClient() + if pgClient == nil { + logger.Error(ctx, "database connection not found in context") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "database connection error", + }) + return + } + + // Query task from database + asyncTask, err := database.GetAsyncTaskByID(ctx, pgClient, taskID) + if err != nil { + if err == gorm.ErrRecordNotFound { + logger.Error(ctx, "async task not found", "task_id", taskID) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusNotFound, + Msg: "task not found", + }) + return + } + logger.Error(ctx, "failed to query async task from database", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to query task", + }) + return + } + + // Query task result from database + taskResult, err := database.GetAsyncTaskResult(ctx, pgClient, taskID) + if err != nil { + logger.Error(ctx, "failed to query async task result from database", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to query task result", + }) + return + } + + // Convert to response format + responseTask := network.AsyncTaskResult{ + TaskID: asyncTask.TaskID, + TaskType: string(asyncTask.TaskType), + Status: string(asyncTask.Status), + CreatedAt: asyncTask.CreatedAt, + FinishedAt: asyncTask.FinishedAt, + Progress: asyncTask.Progress, + } + + // Add result or error information if available + if taskResult != nil { + if taskResult.Result != nil { + responseTask.Result = map[string]any(taskResult.Result) + } + if taskResult.ErrorCode != nil { + responseTask.ErrorCode = taskResult.ErrorCode + } + if taskResult.ErrorMessage != nil { + responseTask.ErrorMessage = taskResult.ErrorMessage + } + if taskResult.ErrorDetail != nil { + responseTask.ErrorDetail = map[string]any(taskResult.ErrorDetail) + } + } + + // Return success response + c.JSON(http.StatusOK, network.SuccessResponse{ + Code: 2000, + Msg: "query completed", + Payload: responseTask, + }) +} diff --git a/handler/async_task_result_query_handler.go b/handler/async_task_result_query_handler.go new file mode 100644 index 0000000..ca76c83 --- /dev/null +++ b/handler/async_task_result_query_handler.go @@ -0,0 +1,144 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "net/http" + + "modelRT/database" + "modelRT/logger" + "modelRT/network" + "modelRT/orm" + + "github.com/gin-gonic/gin" + "github.com/gofrs/uuid" +) + +// AsyncTaskResultQueryHandler handles querying of asynchronous task results +// @Summary 查询异步任务结果 +// @Description 根据任务ID列表查询异步任务的状态和结果 +// @Tags AsyncTask +// @Accept json +// @Produce json +// @Param task_ids query string true "任务ID列表，用逗号分隔" +// @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskResultQueryResponse} "查询成功" +// @Failure 400 {object} network.FailureResponse "请求参数错误" +// @Failure 500 {object} network.FailureResponse "服务器内部错误" +// @Router /task/async/results [get] +func AsyncTaskResultQueryHandler(c *gin.Context) { + ctx := c.Request.Context() + + // Parse task IDs from query parameter + taskIDsParam := c.Query("task_ids") + if taskIDsParam == "" { + logger.Error(ctx, "task_ids parameter is required") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "task_ids parameter is required", + }) + return + } + + // Parse comma-separated task IDs + var taskIDs []uuid.UUID + taskIDStrs := splitCommaSeparated(taskIDsParam) + for _, taskIDStr := range taskIDStrs { + taskID, err := uuid.FromString(taskIDStr) + if err != nil { + logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid task ID format", + }) + return + } + taskIDs = append(taskIDs, taskID) + } + + if len(taskIDs) == 0 { + logger.Error(ctx, "no valid task IDs provided") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "no valid task IDs provided", + }) + return + } + + pgClient := database.GetPostgresDBClient() + if pgClient == nil { + logger.Error(ctx, "database connection not found in context") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "database connection error", + }) + return + } + + // Query tasks from database + asyncTasks, err := database.GetAsyncTasksByIDs(ctx, pgClient, taskIDs) + if err != nil { + logger.Error(ctx, "failed to query async tasks from database", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to query tasks", + }) + return + } + + // Query task results from database + taskResults, err := database.GetAsyncTaskResults(ctx, pgClient, taskIDs) + if err != nil { + logger.Error(ctx, "failed to query async task results from database", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to query task results", + }) + return + } + + // Create a map of task results for easy lookup + taskResultMap := make(map[uuid.UUID]orm.AsyncTaskResult) + for _, result := range taskResults { + taskResultMap[result.TaskID] = result + } + + // Convert to response format + var responseTasks []network.AsyncTaskResult + for _, asyncTask := range asyncTasks { + taskResult := network.AsyncTaskResult{ + TaskID: asyncTask.TaskID, + TaskType: string(asyncTask.TaskType), + Status: string(asyncTask.Status), + CreatedAt: asyncTask.CreatedAt, + FinishedAt: asyncTask.FinishedAt, + Progress: asyncTask.Progress, + } + + // Add result or error information if available + if result, exists := taskResultMap[asyncTask.TaskID]; exists { + if result.Result != nil { + taskResult.Result = map[string]any(result.Result) + } + if result.ErrorCode != nil { + taskResult.ErrorCode = result.ErrorCode + } + if result.ErrorMessage != nil { + taskResult.ErrorMessage = result.ErrorMessage + } + if result.ErrorDetail != nil { + taskResult.ErrorDetail = map[string]any(result.ErrorDetail) + } + } + + responseTasks = append(responseTasks, taskResult) + } + + // Return success response + c.JSON(http.StatusOK, network.SuccessResponse{ + Code: 2000, + Msg: "query completed", + Payload: network.AsyncTaskResultQueryResponse{ + Total: len(responseTasks), + Tasks: responseTasks, + }, + }) +} diff --git a/handler/async_task_status_update_handler.go b/handler/async_task_status_update_handler.go new file mode 100644 index 0000000..caeab2b --- /dev/null +++ b/handler/async_task_status_update_handler.go @@ -0,0 +1,89 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "net/http" + + "modelRT/database" + "modelRT/logger" + "modelRT/network" + "modelRT/orm" + + "github.com/gin-gonic/gin" +) + +// AsyncTaskStatusUpdateHandler handles updating task status (internal use, not exposed via API) +func AsyncTaskStatusUpdateHandler(c *gin.Context) { + ctx := c.Request.Context() + var request network.AsyncTaskStatusUpdate + + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "failed to unmarshal async task status update request", "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid request parameters", + }) + return + } + + // Validate status + validStatus := map[string]bool{ + string(orm.AsyncTaskStatusSubmitted): true, + string(orm.AsyncTaskStatusRunning): true, + string(orm.AsyncTaskStatusCompleted): true, + string(orm.AsyncTaskStatusFailed): true, + } + + if !validStatus[request.Status] { + logger.Error(ctx, "invalid task status", "status", request.Status) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusBadRequest, + Msg: "invalid task status", + }) + return + } + + pgClient := database.GetPostgresDBClient() + if pgClient == nil { + logger.Error(ctx, "database connection not found in context") + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "database connection error", + }) + return + } + + // Update task status + status := orm.AsyncTaskStatus(request.Status) + err := database.UpdateAsyncTaskStatus(ctx, pgClient, request.TaskID, status) + if err != nil { + logger.Error(ctx, "failed to update async task status", "task_id", request.TaskID, "status", request.Status, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to update task status", + }) + return + } + + // If task is completed or failed, update finished_at timestamp + if request.Status == string(orm.AsyncTaskStatusCompleted) { + err = database.CompleteAsyncTask(ctx, pgClient, request.TaskID, request.Timestamp) + } else if request.Status == string(orm.AsyncTaskStatusFailed) { + err = database.FailAsyncTask(ctx, pgClient, request.TaskID, request.Timestamp) + } + + if err != nil { + logger.Error(ctx, "failed to update async task completion timestamp", "task_id", request.TaskID, "error", err) + c.JSON(http.StatusOK, network.FailureResponse{ + Code: http.StatusInternalServerError, + Msg: "failed to update task completion timestamp", + }) + return + } + + c.JSON(http.StatusOK, network.SuccessResponse{ + Code: 2000, + Msg: "task status updated successfully", + Payload: nil, + }) +} From 809e1cd87d42819fbcf27ddd97006e4211b3cc82 Mon Sep 17 00:00:00 2001 From: douxu Date: Wed, 22 Apr 2026 17:20:26 +0800 Subject: [PATCH 29/43] Refactor: extract task constants to dedicated constants package - Add constants/task.go with centralized task-related constants - Task priority levels (default, high, low) - Task queue configuration (exchange, queue, routing key) - Task message settings (max priority, TTL) - Task retry settings (max retries, delays) - Test task settings (sleep duration, max limit) - Update task-related files to use constants from constants package: - handler/async_task_create_handler.go - task/queue_message.go - task/queue_producer.go - task/retry_manager.go - task/test_task.go - task/types.go (add TypeTest) - task/worker.go --- constants/task.go | 54 ++++++++ handler/async_task_create_handler.go | 152 ++++++--------------- handler/async_task_result_query_handler.go | 38 ++++++ task/queue_message.go | 33 ++--- task/queue_producer.go | 70 ++++------ task/retry_manager.go | 17 +-- task/test_task.go | 15 +- task/types.go | 1 + task/worker.go | 31 +++-- 9 files changed, 210 insertions(+), 201 deletions(-) create mode 100644 constants/task.go diff --git a/constants/task.go b/constants/task.go new file mode 100644 index 0000000..bcdd09b --- /dev/null +++ b/constants/task.go @@ -0,0 +1,54 @@ +// Package constants defines task-related constants for the async task system +package constants + +import "time" + +// Task priority levels +const ( + // TaskPriorityDefault is the default priority level for tasks + TaskPriorityDefault = 5 + // TaskPriorityHigh represents high priority tasks + TaskPriorityHigh = 10 + // TaskPriorityLow represents low priority tasks + TaskPriorityLow = 1 +) + +// Task queue configuration +const ( + // TaskExchangeName is the name of the exchange for task routing + TaskExchangeName = "modelrt.tasks.exchange" + // TaskQueueName is the name of the main task queue + TaskQueueName = "modelrt.tasks.queue" + // TaskRoutingKey is the routing key for task messages + TaskRoutingKey = "modelrt.task" +) + +// Task message settings +const ( + // TaskMaxPriority is the maximum priority level for tasks (0-10) + TaskMaxPriority = 10 + // TaskDefaultMessageTTL is the default time-to-live for task messages (24 hours) + TaskDefaultMessageTTL = 24 * time.Hour +) + +// Task retry settings +const ( + // TaskRetryMaxDefault is the default maximum number of retry attempts + TaskRetryMaxDefault = 3 + // TaskRetryInitialDelayDefault is the default initial delay for exponential backoff + TaskRetryInitialDelayDefault = 1 * time.Second + // TaskRetryMaxDelayDefault is the default maximum delay for exponential backoff + TaskRetryMaxDelayDefault = 5 * time.Minute + // TaskRetryRandomFactorDefault is the default random factor for jitter (10%) + TaskRetryRandomFactorDefault = 0.1 + // TaskRetryFixedDelayDefault is the default delay for fixed retry strategy + TaskRetryFixedDelayDefault = 5 * time.Second +) + +// Test task settings +const ( + // TestTaskSleepDurationDefault is the default sleep duration for test tasks (60 seconds) + TestTaskSleepDurationDefault = 60 + // TestTaskSleepDurationMax is the maximum allowed sleep duration for test tasks (1 hour) + TestTaskSleepDurationMax = 3600 +) diff --git a/handler/async_task_create_handler.go b/handler/async_task_create_handler.go index 630b594..2726ce5 100644 --- a/handler/async_task_create_handler.go +++ b/handler/async_task_create_handler.go @@ -2,10 +2,8 @@ package handler import ( - "net/http" - "strings" - "modelRT/config" + "modelRT/constants" "modelRT/database" "modelRT/logger" "modelRT/network" @@ -13,7 +11,6 @@ import ( "modelRT/task" "github.com/gin-gonic/gin" - "gorm.io/gorm" ) // AsyncTaskCreateHandler handles creation of asynchronous tasks @@ -32,59 +29,44 @@ func AsyncTaskCreateHandler(c *gin.Context) { var request network.AsyncTaskCreateRequest if err := c.ShouldBindJSON(&request); err != nil { - logger.Error(ctx, "failed to unmarshal async task create request", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid request parameters", - }) + logger.Error(ctx, "unmarshal async task create request failed", "error", err) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid request parameters", nil) return } - // Validate task type + // validate task type if !orm.IsValidAsyncTaskType(request.TaskType) { - logger.Error(ctx, "invalid task type", "task_type", request.TaskType) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid task type", - }) + logger.Error(ctx, "check task type invalid", "task_type", request.TaskType) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid task type", nil) return } - // Validate task parameters based on task type + // validate task parameters based on task type if !validateTaskParams(request.TaskType, request.Params) { - logger.Error(ctx, "invalid task parameters", "task_type", request.TaskType, "params", request.Params) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid task parameters", - }) + logger.Error(ctx, "check task parameters invalid", "task_type", request.TaskType, "params", request.Params) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid task parameters", nil) return } pgClient := database.GetPostgresDBClient() if pgClient == nil { logger.Error(ctx, "database connection not found in context") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "database connection error", - }) + renderRespFailure(c, constants.RespCodeServerError, "database connection error", nil) return } - // Create task in database + // create task in database taskType := orm.AsyncTaskType(request.TaskType) params := orm.JSONMap(request.Params) asyncTask, err := database.CreateAsyncTask(ctx, pgClient, taskType, params) if err != nil { - logger.Error(ctx, "failed to create async task in database", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to create task", - }) + logger.Error(ctx, "create async task in database failed", "error", err) + renderRespFailure(c, constants.RespCodeServerError, "failed to create task", nil) return } - // Send task to message queue + // send task to message queue cfg, exists := c.Get("config") if !exists { logger.Warn(ctx, "Configuration not found in context, skipping queue publishing") @@ -92,41 +74,36 @@ func AsyncTaskCreateHandler(c *gin.Context) { modelRTConfig := cfg.(config.ModelRTConfig) ctx := c.Request.Context() - // Create queue producer + // create queue producer + // TODO 像实时计算一样使用 channel 代替 producer, err := task.NewQueueProducer(ctx, modelRTConfig.RabbitMQConfig) if err != nil { - logger.Error(ctx, "Failed to create queue producer", "error", err) - // Continue without queue publishing - } else { - defer producer.Close() - - // Publish task to queue - taskType := task.TaskType(request.TaskType) - priority := 5 // Default priority - - if err := producer.PublishTaskWithRetry(ctx, asyncTask.TaskID, taskType, priority, 3); err != nil { - logger.Error(ctx, "Failed to publish task to queue", - "task_id", asyncTask.TaskID, - "error", err) - // Log error but don't affect task creation response - } else { - logger.Info(ctx, "Task published to queue successfully", - "task_id", asyncTask.TaskID, - "queue", task.TaskQueueName) - } + logger.Error(ctx, "create rabbitMQ queue producer failed", "error", err) + renderRespFailure(c, constants.RespCodeServerError, "create rabbitMQ queue producer failed", nil) + return } + defer producer.Close() + + // publish task to queue + taskType := task.TaskType(request.TaskType) + priority := 5 // Default priority + + if err := producer.PublishTaskWithRetry(ctx, asyncTask.TaskID, taskType, priority, 3); err != nil { + logger.Error(ctx, "publish task to rabbitMQ queue failed", + "task_id", asyncTask.TaskID, "error", err) + renderRespFailure(c, constants.RespCodeServerError, "publish task to rabbitMQ queue failed", nil) + return + } + logger.Info(ctx, "published task to rabbitMQ queue successfully", + "task_id", asyncTask.TaskID, "queue", constants.TaskQueueName) + } - logger.Info(ctx, "async task created successfully", "task_id", asyncTask.TaskID, "task_type", request.TaskType) + logger.Info(ctx, "async task created success", "task_id", asyncTask.TaskID, "task_type", request.TaskType) - // Return success response - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: 2000, - Msg: "task created successfully", - Payload: network.AsyncTaskCreateResponse{ - TaskID: asyncTask.TaskID, - }, - }) + // return success response + payload := genAsyncTaskCreatePayload(asyncTask.TaskID.String()) + renderRespSuccess(c, constants.RespCodeSuccess, "task created successfully", payload) } func validateTaskParams(taskType string, params map[string]any) bool { @@ -189,54 +166,9 @@ func validateTestTaskParams(params map[string]any) bool { return true } -func splitCommaSeparated(s string) []string { - var result []string - var current strings.Builder - inQuotes := false - escape := false - - for _, ch := range s { - if escape { - current.WriteRune(ch) - escape = false - continue - } - - switch ch { - case '\\': - escape = true - case '"': - inQuotes = !inQuotes - case ',': - if !inQuotes { - result = append(result, strings.TrimSpace(current.String())) - current.Reset() - } else { - current.WriteRune(ch) - } - default: - current.WriteRune(ch) - } +func genAsyncTaskCreatePayload(taskID string) map[string]any { + payload := map[string]any{ + "task_id": taskID, } - - if current.Len() > 0 { - result = append(result, strings.TrimSpace(current.String())) - } - - return result -} - -func getDBFromContext(c *gin.Context) *gorm.DB { - // Try to get database connection from context - // This should be set by middleware - if db, exists := c.Get("db"); exists { - if gormDB, ok := db.(*gorm.DB); ok { - return gormDB - } - } - - // Fallback to global database connection - // This should be implemented based on your application's database setup - // For now, return nil - actual implementation should retrieve from application context - return nil + return payload } diff --git a/handler/async_task_result_query_handler.go b/handler/async_task_result_query_handler.go index ca76c83..12c7d67 100644 --- a/handler/async_task_result_query_handler.go +++ b/handler/async_task_result_query_handler.go @@ -3,6 +3,7 @@ package handler import ( "net/http" + "strings" "modelRT/database" "modelRT/logger" @@ -142,3 +143,40 @@ func AsyncTaskResultQueryHandler(c *gin.Context) { }, }) } + +func splitCommaSeparated(s string) []string { + var result []string + var current strings.Builder + inQuotes := false + escape := false + + for _, ch := range s { + if escape { + current.WriteRune(ch) + escape = false + continue + } + + switch ch { + case '\\': + escape = true + case '"': + inQuotes = !inQuotes + case ',': + if !inQuotes { + result = append(result, strings.TrimSpace(current.String())) + current.Reset() + } else { + current.WriteRune(ch) + } + default: + current.WriteRune(ch) + } + } + + if current.Len() > 0 { + result = append(result, strings.TrimSpace(current.String())) + } + + return result +} diff --git a/task/queue_message.go b/task/queue_message.go index 7ea0164..6ce8028 100644 --- a/task/queue_message.go +++ b/task/queue_message.go @@ -3,24 +3,17 @@ package task import ( "encoding/json" + "modelRT/constants" + "github.com/gofrs/uuid" ) -// DefaultPriority is the default task priority -const DefaultPriority = 5 - -// HighPriority represents high priority tasks -const HighPriority = 10 - -// LowPriority represents low priority tasks -const LowPriority = 1 - // TaskQueueMessage defines minimal message structure for RabbitMQ/Redis queue dispatch // This struct is designed to be lightweight for efficient message transport type TaskQueueMessage struct { TaskID uuid.UUID `json:"task_id"` TaskType TaskType `json:"task_type"` - Priority int `json:"priority,omitempty"` // Optional, defaults to DefaultPriority + Priority int `json:"priority,omitempty"` // Optional, defaults to constants.TaskPriorityDefault } // NewTaskQueueMessage creates a new TaskQueueMessage with default priority @@ -28,7 +21,7 @@ func NewTaskQueueMessage(taskID uuid.UUID, taskType TaskType) *TaskQueueMessage return &TaskQueueMessage{ TaskID: taskID, TaskType: taskType, - Priority: DefaultPriority, + Priority: constants.TaskPriorityDefault, } } @@ -41,12 +34,12 @@ func NewTaskQueueMessageWithPriority(taskID uuid.UUID, taskType TaskType, priori } } -// ToJSON converts the TaskQueueMessage to JSON bytes +// ToJSON converts TaskQueueMessage to JSON bytes func (m *TaskQueueMessage) ToJSON() ([]byte, error) { return json.Marshal(m) } -// Validate checks if the TaskQueueMessage is valid +// Validate checks if TaskQueueMessage is valid func (m *TaskQueueMessage) Validate() bool { // Check if TaskID is valid (not nil UUID) if m.TaskID == uuid.Nil { @@ -55,25 +48,25 @@ func (m *TaskQueueMessage) Validate() bool { // Check if TaskType is valid switch m.TaskType { - case TypeTopologyAnalysis, TypeEventAnalysis, TypeBatchImport: + case TypeTopologyAnalysis, TypeEventAnalysis, TypeBatchImport, TypeTest: return true default: return false } } -// SetPriority sets the priority of the task queue message with validation +// SetPriority sets priority of task queue message with validation func (m *TaskQueueMessage) SetPriority(priority int) { - if priority < LowPriority { - priority = LowPriority + if priority < constants.TaskPriorityLow { + priority = constants.TaskPriorityLow } - if priority > HighPriority { - priority = HighPriority + if priority > constants.TaskPriorityHigh { + priority = constants.TaskPriorityHigh } m.Priority = priority } -// GetPriority returns the priority of the task queue message +// GetPriority returns priority of task queue message func (m *TaskQueueMessage) GetPriority() int { return m.Priority } diff --git a/task/queue_producer.go b/task/queue_producer.go index b9f2df1..650a2bf 100644 --- a/task/queue_producer.go +++ b/task/queue_producer.go @@ -8,6 +8,7 @@ import ( "time" "modelRT/config" + "modelRT/constants" "modelRT/logger" "modelRT/mq" @@ -15,19 +16,6 @@ import ( amqp "github.com/rabbitmq/amqp091-go" ) -const ( - // TaskExchangeName is the name of the exchange for task routing - TaskExchangeName = "modelrt.tasks.exchange" - // TaskQueueName is the name of the main task queue - TaskQueueName = "modelrt.tasks.queue" - // TaskRoutingKey is the routing key for task messages - TaskRoutingKey = "modelrt.task" - // MaxPriority is the maximum priority level for tasks (0-10) - MaxPriority = 10 - // DefaultMessageTTL is the default time-to-live for task messages (24 hours) - DefaultMessageTTL = 24 * time.Hour -) - // QueueProducer handles publishing tasks to RabbitMQ type QueueProducer struct { conn *amqp.Connection @@ -67,13 +55,13 @@ func NewQueueProducer(ctx context.Context, cfg config.RabbitMQConfig) (*QueuePro func (p *QueueProducer) declareInfrastructure() error { // Declare durable direct exchange err := p.ch.ExchangeDeclare( - TaskExchangeName, // name - "direct", // type - true, // durable - false, // auto-deleted - false, // internal - false, // no-wait - nil, // arguments + constants.TaskExchangeName, // name + "direct", // type + true, // durable + false, // auto-deleted + false, // internal + false, // no-wait + nil, // arguments ) if err != nil { return fmt.Errorf("failed to declare exchange: %w", err) @@ -81,14 +69,14 @@ func (p *QueueProducer) declareInfrastructure() error { // Declare durable queue with priority support and message TTL _, err = p.ch.QueueDeclare( - TaskQueueName, // name - true, // durable - false, // delete when unused - false, // exclusive - false, // no-wait + constants.TaskQueueName, // name + true, // durable + false, // delete when unused + false, // exclusive + false, // no-wait amqp.Table{ - "x-max-priority": MaxPriority, // support priority levels 0-10 - "x-message-ttl": DefaultMessageTTL.Milliseconds(), // message TTL + "x-max-priority": constants.TaskMaxPriority, // support priority levels 0-10 + "x-message-ttl": constants.TaskDefaultMessageTTL.Milliseconds(), // message TTL }, ) if err != nil { @@ -97,11 +85,11 @@ func (p *QueueProducer) declareInfrastructure() error { // Bind queue to exchange err = p.ch.QueueBind( - TaskQueueName, // queue name - TaskRoutingKey, // routing key - TaskExchangeName, // exchange name - false, // no-wait - nil, // arguments + constants.TaskQueueName, // queue name + constants.TaskRoutingKey, // routing key + constants.TaskExchangeName, // exchange name + false, // no-wait + nil, // arguments ) if err != nil { return fmt.Errorf("failed to bind queue: %w", err) @@ -141,10 +129,10 @@ func (p *QueueProducer) PublishTask(ctx context.Context, taskID uuid.UUID, taskT // Publish to exchange err = p.ch.PublishWithContext( ctx, - TaskExchangeName, // exchange - TaskRoutingKey, // routing key - false, // mandatory - false, // immediate + constants.TaskExchangeName, // exchange + constants.TaskRoutingKey, // routing key + false, // mandatory + false, // immediate publishing, ) if err != nil { @@ -155,7 +143,7 @@ func (p *QueueProducer) PublishTask(ctx context.Context, taskID uuid.UUID, taskT "task_id", taskID.String(), "task_type", taskType, "priority", priority, - "queue", TaskQueueName, + "queue", constants.TaskQueueName, ) return nil @@ -205,14 +193,14 @@ func (p *QueueProducer) Close() error { // GetQueueInfo returns information about the task queue func (p *QueueProducer) GetQueueInfo() (*amqp.Queue, error) { queue, err := p.ch.QueueDeclarePassive( - TaskQueueName, // name + constants.TaskQueueName, // name true, // durable false, // delete when unused false, // exclusive false, // no-wait amqp.Table{ - "x-max-priority": MaxPriority, - "x-message-ttl": DefaultMessageTTL.Milliseconds(), + "x-max-priority": constants.TaskMaxPriority, + "x-message-ttl": constants.TaskDefaultMessageTTL.Milliseconds(), }, ) if err != nil { @@ -223,5 +211,5 @@ func (p *QueueProducer) GetQueueInfo() (*amqp.Queue, error) { // PurgeQueue removes all messages from the task queue func (p *QueueProducer) PurgeQueue() (int, error) { - return p.ch.QueuePurge(TaskQueueName, false) + return p.ch.QueuePurge(constants.TaskQueueName, false) } \ No newline at end of file diff --git a/task/retry_manager.go b/task/retry_manager.go index ddd7db5..be70c3a 100644 --- a/task/retry_manager.go +++ b/task/retry_manager.go @@ -8,12 +8,13 @@ import ( "strings" "time" + "modelRT/constants" "modelRT/logger" ) // RetryStrategy defines the interface for task retry strategies type RetryStrategy interface { - // ShouldRetry determines if a task should be retried and returns the delay before next retry + // ShouldRetry determines if a task should be retried and returns delay before next retry ShouldRetry(ctx context.Context, taskID string, retryCount int, lastError error) (bool, time.Duration) // GetMaxRetries returns the maximum number of retry attempts GetMaxRetries() int @@ -98,7 +99,7 @@ func (s *ExponentialBackoffRetry) ShouldRetry(ctx context.Context, taskID string return true, delay } -// GetMaxRetries returns the maximum number of retry attempts +// GetMaxRetries returns maximum number of retry attempts func (s *ExponentialBackoffRetry) GetMaxRetries() int { return s.MaxRetries } @@ -151,7 +152,7 @@ func (s *FixedDelayRetry) ShouldRetry(ctx context.Context, taskID string, retryC return true, delay } -// GetMaxRetries returns the maximum number of retry attempts +// GetMaxRetries returns maximum number of retry attempts func (s *FixedDelayRetry) GetMaxRetries() int { return s.MaxRetries } @@ -177,10 +178,10 @@ func (s *NoRetryStrategy) GetMaxRetries() int { // DefaultRetryStrategy returns the default retry strategy (exponential backoff) func DefaultRetryStrategy() RetryStrategy { return NewExponentialBackoffRetry( - 3, // max retries - 1*time.Second, // initial delay - 5*time.Minute, // max delay - 0.1, // random factor (10% jitter) + constants.TaskRetryMaxDefault, // max retries + constants.TaskRetryInitialDelayDefault, // initial delay + constants.TaskRetryMaxDelayDefault, // max delay + constants.TaskRetryRandomFactorDefault, // random factor (10% jitter) ) } @@ -216,4 +217,4 @@ func IsRetryableError(err error) bool { } return false -} \ No newline at end of file +} diff --git a/task/test_task.go b/task/test_task.go index 2d60bd4..4b38ac9 100644 --- a/task/test_task.go +++ b/task/test_task.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "modelRT/constants" "modelRT/database" "modelRT/logger" "modelRT/orm" @@ -17,7 +18,7 @@ import ( // TestTaskParams defines parameters for test task type TestTaskParams struct { // SleepDuration specifies how long the task should sleep (in seconds) - // Default is 60 seconds as per requirement + // Default is constants.TestTaskSleepDurationDefault seconds as per requirement SleepDuration int `json:"sleep_duration"` // Message is a custom message to include in the result Message string `json:"message,omitempty"` @@ -25,14 +26,14 @@ type TestTaskParams struct { // Validate checks if test task parameters are valid func (p *TestTaskParams) Validate() error { - // Default to 60 seconds if not specified + // Default to constants.TestTaskSleepDurationDefault seconds if not specified if p.SleepDuration <= 0 { - p.SleepDuration = 60 + p.SleepDuration = constants.TestTaskSleepDurationDefault } // Validate max duration (max 1 hour) - if p.SleepDuration > 3600 { - return fmt.Errorf("sleep duration cannot exceed 3600 seconds (1 hour)") + if p.SleepDuration > constants.TestTaskSleepDurationMax { + return fmt.Errorf("sleep duration cannot exceed %d seconds (1 hour)", constants.TestTaskSleepDurationMax) } return nil @@ -90,7 +91,7 @@ func (t *TestTask) Execute(ctx context.Context, taskID uuid.UUID, db *gorm.DB) e return fmt.Errorf("invalid parameter type for TestTask") } - logger.Info(ctx, "Starting test task execution", + logger.Info(ctx, "Starting test task executionser", "task_id", taskID, "sleep_duration_seconds", params.SleepDuration, "message", params.Message, @@ -149,7 +150,7 @@ func (h *TestTaskHandler) Execute(ctx context.Context, taskID uuid.UUID, taskTyp // Fetch task parameters from database asyncTask, err := database.GetAsyncTaskByID(ctx, db, taskID) if err != nil { - return fmt.Errorf("failed to fetch task: %w", err) + return fmt.Errorf("failed toser fetch task: %w", err) } // Convert params map to TestTaskParams diff --git a/task/types.go b/task/types.go index 8ea5c73..be1f4f5 100644 --- a/task/types.go +++ b/task/types.go @@ -20,6 +20,7 @@ const ( TypeTopologyAnalysis TaskType = "TOPOLOGY_ANALYSIS" TypeEventAnalysis TaskType = "EVENT_ANALYSIS" TypeBatchImport TaskType = "BATCH_IMPORT" + TypeTest TaskType = "TEST" ) type Task struct { diff --git a/task/worker.go b/task/worker.go index 8388f8e..03f3728 100644 --- a/task/worker.go +++ b/task/worker.go @@ -9,6 +9,7 @@ import ( "time" "modelRT/config" + "modelRT/constants" "modelRT/database" "modelRT/logger" "modelRT/mq" @@ -112,14 +113,14 @@ func NewTaskWorker(ctx context.Context, cfg WorkerConfig, db *gorm.DB, rabbitCfg // Declare queue (ensure it exists with proper arguments) _, err = ch.QueueDeclare( - TaskQueueName, // name - true, // durable - false, // delete when unused - false, // exclusive - false, // no-wait + constants.TaskQueueName, // name + true, // durable + false, // delete when unused + false, // exclusive + false, // no-wait amqp.Table{ - "x-max-priority": MaxPriority, - "x-message-ttl": DefaultMessageTTL.Milliseconds(), + "x-max-priority": constants.TaskMaxPriority, + "x-message-ttl": constants.TaskDefaultMessageTTL.Milliseconds(), }, ) if err != nil { @@ -198,7 +199,7 @@ func (w *TaskWorker) consumerLoop(consumerID int) { // Consume messages from the queue msgs, err := w.ch.Consume( - TaskQueueName, // queue + constants.TaskQueueName, // queue fmt.Sprintf("worker-%d", consumerID), // consumer tag false, // auto-ack false, // exclusive @@ -462,14 +463,14 @@ func (w *TaskWorker) checkHealth() { // Update queue depth queue, err := w.ch.QueueDeclarePassive( - TaskQueueName, // name - true, // durable - false, // delete when unused - false, // exclusive - false, // no-wait + constants.TaskQueueName, // name + true, // durable + false, // delete when unused + false, // exclusive + false, // no-wait amqp.Table{ - "x-max-priority": MaxPriority, - "x-message-ttl": DefaultMessageTTL.Milliseconds(), + "x-max-priority": constants.TaskMaxPriority, + "x-message-ttl": constants.TaskDefaultMessageTTL.Milliseconds(), }, ) if err == nil { From 03bd058558dc30d3f31060fc9ceae653748d25bf Mon Sep 17 00:00:00 2001 From: douxu Date: Thu, 23 Apr 2026 16:48:32 +0800 Subject: [PATCH 30/43] =?UTF-8?q?feat:=20implement=20end-to-end=20distribu?= =?UTF-8?q?ted=20tracing=20for=20HTTP=20and=20async=20tasks=20=20=20-=20in?= =?UTF-8?q?troduce=20typed=20traceCtxKey=20to=20prevent=20context=20key=20?= =?UTF-8?q?collisions=20(staticcheck=20fix)=20=20=20-=20inject=20B3=20trac?= =?UTF-8?q?e=20values=20into=20c.Request.Context()=20in=20StartTrace=20mid?= =?UTF-8?q?dleware=20=20=20=20=20so=20handlers=20using=20c.Request.Context?= =?UTF-8?q?()=20carry=20trace=20info=20=20=20-=20create=20startup=20trace?= =?UTF-8?q?=20context=20in=20main.go,=20replacing=20context.TODO()=20=20?= =?UTF-8?q?=20-=20propagate=20HTTP=20traceID/spanID=20through=20TaskQueueM?= =?UTF-8?q?essage=20into=20RabbitMQ=20=20=20=20=20worker,=20linking=20HTTP?= =?UTF-8?q?=20request=20=E2=86=92=20publish=20=E2=86=92=20execution=20on?= =?UTF-8?q?=20the=20same=20traceID=20=20=20-=20fix=20GORM=20logger=20null?= =?UTF-8?q?=20traceID=20by=20binding=20ctx=20to=20AutoMigrate=20and=20quer?= =?UTF-8?q?ies=20=20=20=20=20via=20db.WithContext(ctx)=20=20=20-=20thread?= =?UTF-8?q?=20ctx=20through=20handler=20factory=20to=20fix=20null=20traceI?= =?UTF-8?q?D=20in=20startup=20logs=20=20=20-=20replace=20per-request=20Rab?= =?UTF-8?q?bitMQ=20producer=20with=20channel-based=20=20=20=20=20PushTaskT?= =?UTF-8?q?oRabbitMQ=20goroutine;=20restrict=20Swagger=20to=20non-producti?= =?UTF-8?q?on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- constants/trace.go | 10 +++++++ database/postgres_init.go | 9 +++--- handler/async_task_create_handler.go | 42 +++++++-------------------- logger/logger.go | 29 +++++++++++++------ main.go | 28 +++++++----------- middleware/config_middleware.go | 3 +- middleware/limiter.go | 1 + middleware/panic_recover.go | 1 + middleware/token.go | 1 + middleware/trace.go | 11 ++++++- model/recommend_islocal_cache.go | 2 +- task/handler_factory.go | 22 +++++++------- task/initializer.go | 6 ++-- task/queue_message.go | 2 ++ task/queue_producer.go | 43 ++++++++++++++++++++++++++++ task/worker.go | 11 +++++++ 16 files changed, 142 insertions(+), 79 deletions(-) diff --git a/constants/trace.go b/constants/trace.go index 14f52ed..ab5c5df 100644 --- a/constants/trace.go +++ b/constants/trace.go @@ -7,3 +7,13 @@ const ( HeaderSpanID = "X-B3-SpanId" HeaderParentSpanID = "X-B3-ParentSpanId" ) + +// traceCtxKey is an unexported type for context keys to avoid collisions with other packages. +type traceCtxKey string + +// Typed context keys for trace values — use these with context.WithValue / ctx.Value. +var ( + CtxKeyTraceID = traceCtxKey(HeaderTraceID) + CtxKeySpanID = traceCtxKey(HeaderSpanID) + CtxKeyParentSpanID = traceCtxKey(HeaderParentSpanID) +) diff --git a/database/postgres_init.go b/database/postgres_init.go index d2babd4..c2e0ed0 100644 --- a/database/postgres_init.go +++ b/database/postgres_init.go @@ -2,6 +2,7 @@ package database import ( + "context" "sync" "modelRT/logger" @@ -22,22 +23,22 @@ func GetPostgresDBClient() *gorm.DB { } // InitPostgresDBInstance return instance of PostgresDB client -func InitPostgresDBInstance(PostgresDBURI string) *gorm.DB { +func InitPostgresDBInstance(ctx context.Context, PostgresDBURI string) *gorm.DB { postgresOnce.Do(func() { - _globalPostgresClient = initPostgresDBClient(PostgresDBURI) + _globalPostgresClient = initPostgresDBClient(ctx, PostgresDBURI) }) return _globalPostgresClient } // initPostgresDBClient return successfully initialized PostgresDB client -func initPostgresDBClient(PostgresDBURI string) *gorm.DB { +func initPostgresDBClient(ctx context.Context, PostgresDBURI string) *gorm.DB { db, err := gorm.Open(postgres.Open(PostgresDBURI), &gorm.Config{Logger: logger.NewGormLogger()}) if err != nil { panic(err) } // Auto migrate async task tables - err = db.AutoMigrate( + err = db.WithContext(ctx).AutoMigrate( &orm.AsyncTask{}, &orm.AsyncTaskResult{}, ) diff --git a/handler/async_task_create_handler.go b/handler/async_task_create_handler.go index 2726ce5..6bf8a72 100644 --- a/handler/async_task_create_handler.go +++ b/handler/async_task_create_handler.go @@ -2,7 +2,6 @@ package handler import ( - "modelRT/config" "modelRT/constants" "modelRT/database" "modelRT/logger" @@ -66,38 +65,17 @@ func AsyncTaskCreateHandler(c *gin.Context) { return } - // send task to message queue - cfg, exists := c.Get("config") - if !exists { - logger.Warn(ctx, "Configuration not found in context, skipping queue publishing") - } else { - modelRTConfig := cfg.(config.ModelRTConfig) - ctx := c.Request.Context() - - // create queue producer - // TODO 像实时计算一样使用 channel 代替 - producer, err := task.NewQueueProducer(ctx, modelRTConfig.RabbitMQConfig) - if err != nil { - logger.Error(ctx, "create rabbitMQ queue producer failed", "error", err) - renderRespFailure(c, constants.RespCodeServerError, "create rabbitMQ queue producer failed", nil) - return - } - defer producer.Close() - - // publish task to queue - taskType := task.TaskType(request.TaskType) - priority := 5 // Default priority - - if err := producer.PublishTaskWithRetry(ctx, asyncTask.TaskID, taskType, priority, 3); err != nil { - logger.Error(ctx, "publish task to rabbitMQ queue failed", - "task_id", asyncTask.TaskID, "error", err) - renderRespFailure(c, constants.RespCodeServerError, "publish task to rabbitMQ queue failed", nil) - return - } - logger.Info(ctx, "published task to rabbitMQ queue successfully", - "task_id", asyncTask.TaskID, "queue", constants.TaskQueueName) - + // enqueue task to channel for async publishing to RabbitMQ + msg := task.NewTaskQueueMessageWithPriority(asyncTask.TaskID, task.TaskType(request.TaskType), 5) + // propagate HTTP request trace so the async chain stays on the same traceID + if v, _ := ctx.Value(constants.CtxKeyTraceID).(string); v != "" { + msg.TraceID = v } + if v, _ := ctx.Value(constants.CtxKeySpanID).(string); v != "" { + msg.SpanID = v + } + task.TaskMsgChan <- msg + logger.Info(ctx, "task enqueued to channel", "task_id", asyncTask.TaskID, "queue", constants.TaskQueueName) logger.Info(ctx, "async task created success", "task_id", asyncTask.TaskID, "task_type", request.TaskType) diff --git a/logger/logger.go b/logger/logger.go index 116e3b1..0d4317a 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -12,6 +12,14 @@ import ( "go.uber.org/zap/zapcore" ) +// Logger is the interface returned by New for structured, trace-aware logging. +type Logger interface { + Debug(msg string, kv ...any) + Info(msg string, kv ...any) + Warn(msg string, kv ...any) + Error(msg string, kv ...any) +} + type logger struct { ctx context.Context traceID string @@ -48,7 +56,10 @@ func makeLogFields(ctx context.Context, kv ...any) []zap.Field { kv = append(kv, "unknown") } - kv = append(kv, "traceID", ctx.Value(constants.HeaderTraceID), "spanID", ctx.Value(constants.HeaderSpanID), "parentSpanID", ctx.Value(constants.HeaderParentSpanID)) + traceID, _ := ctx.Value(constants.CtxKeyTraceID).(string) + spanID, _ := ctx.Value(constants.CtxKeySpanID).(string) + parentSpanID, _ := ctx.Value(constants.CtxKeyParentSpanID).(string) + kv = append(kv, "traceID", traceID, "spanID", spanID, "parentSpanID", parentSpanID) funcName, file, line := getLoggerCallerInfo() kv = append(kv, "func", funcName, "file", file, "line", line) @@ -89,16 +100,18 @@ func getLoggerCallerInfo() (funcName, file string, line int) { return } -func New(ctx context.Context) *logger { +// New returns a logger bound to ctx. Trace fields (traceID, spanID, parentSpanID) +// are extracted from ctx using typed keys, and are included in every log entry. +func New(ctx context.Context) Logger { var traceID, spanID, pSpanID string - if ctx.Value("traceID") != nil { - traceID = ctx.Value("traceID").(string) + if v, _ := ctx.Value(constants.CtxKeyTraceID).(string); v != "" { + traceID = v } - if ctx.Value("spanID") != nil { - spanID = ctx.Value("spanID").(string) + if v, _ := ctx.Value(constants.CtxKeySpanID).(string); v != "" { + spanID = v } - if ctx.Value("psapnID") != nil { - pSpanID = ctx.Value("pspanID").(string) + if v, _ := ctx.Value(constants.CtxKeyParentSpanID).(string); v != "" { + pSpanID = v } return &logger{ diff --git a/main.go b/main.go index 198bec0..103eca9 100644 --- a/main.go +++ b/main.go @@ -19,7 +19,6 @@ import ( "modelRT/database" "modelRT/diagram" "modelRT/logger" - "modelRT/middleware" "modelRT/model" "modelRT/mq" "modelRT/pool" @@ -74,7 +73,9 @@ var ( func main() { flag.Parse() - ctx := context.TODO() + startupSpanID := util.GenerateSpanID("startup") + ctx := context.WithValue(context.Background(), constants.CtxKeyTraceID, startupSpanID) + ctx = context.WithValue(ctx, constants.CtxKeySpanID, startupSpanID) configPath := filepath.Join(*modelRTConfigDir, *modelRTConfigName+"."+*modelRTConfigType) if _, err := os.Stat(configPath); os.IsNotExist(err) { @@ -113,7 +114,7 @@ func main() { } // init postgresDBClient - postgresDBClient = database.InitPostgresDBInstance(modelRTConfig.PostgresDBURI) + postgresDBClient = database.InitPostgresDBInstance(ctx, modelRTConfig.PostgresDBURI) defer func() { sqlDB, err := postgresDBClient.DB() @@ -171,8 +172,10 @@ func main() { // async push event to rabbitMQ go mq.PushUpDownLimitEventToRabbitMQ(ctx, mq.MsgChan) + // async push task message to rabbitMQ + go task.PushTaskToRabbitMQ(ctx, modelRTConfig.RabbitMQConfig, task.TaskMsgChan) - postgresDBClient.Transaction(func(tx *gorm.DB) error { + postgresDBClient.WithContext(ctx).Transaction(func(tx *gorm.DB) error { // load circuit diagram from postgres // componentTypeMap, err := database.QueryCircuitDiagramComponentFromDB(cancelCtx, tx, parsePool) // if err != nil { @@ -246,22 +249,11 @@ func main() { AllowCredentials: true, MaxAge: 12 * time.Hour, })) - // Register configuration middleware - engine.Use(middleware.ConfigMiddleware(modelRTConfig)) router.RegisterRoutes(engine, serviceToken) - // Swagger UI - engine.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler)) - - // 注册 Swagger UI 路由 - // docs.SwaggerInfo.BasePath = "/model" - // v1 := engine.Group("/api/v1") - // { - // eg := v1.Group("/example") - // { - // eg.GET("/helloworld", Helloworld) - // } - // } + if modelRTConfig.DeployEnv != constants.ProductionDeployMode { + engine.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler)) + } server := http.Server{ Addr: modelRTConfig.ServiceAddr, diff --git a/middleware/config_middleware.go b/middleware/config_middleware.go index ff56995..9ff5436 100644 --- a/middleware/config_middleware.go +++ b/middleware/config_middleware.go @@ -1,3 +1,4 @@ +// Package middleware define gin framework middlewares package middleware import ( @@ -12,4 +13,4 @@ func ConfigMiddleware(modelRTConfig config.ModelRTConfig) gin.HandlerFunc { c.Set("config", modelRTConfig) c.Next() } -} \ No newline at end of file +} diff --git a/middleware/limiter.go b/middleware/limiter.go index ca8dc53..d1debe1 100644 --- a/middleware/limiter.go +++ b/middleware/limiter.go @@ -1,3 +1,4 @@ +// Package middleware define gin framework middlewares package middleware import ( diff --git a/middleware/panic_recover.go b/middleware/panic_recover.go index e7aa3ac..b8c101f 100644 --- a/middleware/panic_recover.go +++ b/middleware/panic_recover.go @@ -1,3 +1,4 @@ +// Package middleware define gin framework middlewares package middleware import ( diff --git a/middleware/token.go b/middleware/token.go index 06ffa79..6759f40 100644 --- a/middleware/token.go +++ b/middleware/token.go @@ -1,3 +1,4 @@ +// Package middleware define gin framework middlewares package middleware import "github.com/gin-gonic/gin" diff --git a/middleware/trace.go b/middleware/trace.go index 1bb27ca..604a08c 100644 --- a/middleware/trace.go +++ b/middleware/trace.go @@ -1,7 +1,9 @@ +// Package middleware define gin framework middlewares package middleware import ( "bytes" + "context" "io" "strings" "time" @@ -27,6 +29,14 @@ func StartTrace() gin.HandlerFunc { c.Set(constants.HeaderTraceID, traceID) c.Set(constants.HeaderSpanID, spanID) c.Set(constants.HeaderParentSpanID, parentSpanID) + + // also inject into request context so c.Request.Context() carries trace values + reqCtx := c.Request.Context() + reqCtx = context.WithValue(reqCtx, constants.CtxKeyTraceID, traceID) + reqCtx = context.WithValue(reqCtx, constants.CtxKeySpanID, spanID) + reqCtx = context.WithValue(reqCtx, constants.CtxKeyParentSpanID, parentSpanID) + c.Request = c.Request.WithContext(reqCtx) + c.Next() } } @@ -78,7 +88,6 @@ func LogAccess() gin.HandlerFunc { accessLog(c, "access_end", time.Since(start), reqBody, responseLogging) }() c.Next() - return } } diff --git a/model/recommend_islocal_cache.go b/model/recommend_islocal_cache.go index 8bb7cda..a9f54b5 100644 --- a/model/recommend_islocal_cache.go +++ b/model/recommend_islocal_cache.go @@ -22,7 +22,7 @@ func GetNSpathToIsLocalMap(ctx context.Context, db *gorm.DB) (map[string]bool, e var results []ComponentStationRelation nspathMap := make(map[string]bool) - err := db.Table("component"). + err := db.WithContext(ctx).Table("component"). Select("component.nspath, station.is_local"). Joins("join station on component.station_id = station.id"). Scan(&results).Error diff --git a/task/handler_factory.go b/task/handler_factory.go index c472fef..1efdf65 100644 --- a/task/handler_factory.go +++ b/task/handler_factory.go @@ -36,12 +36,12 @@ func NewHandlerFactory() *HandlerFactory { } // RegisterHandler registers a handler for a specific task type -func (f *HandlerFactory) RegisterHandler(taskType TaskType, handler TaskHandler) { +func (f *HandlerFactory) RegisterHandler(ctx context.Context, taskType TaskType, handler TaskHandler) { f.mu.Lock() defer f.mu.Unlock() f.handlers[taskType] = handler - logger.Info(context.Background(), "Handler registered", + logger.Info(ctx, "Handler registered", "task_type", taskType, "handler_name", handler.Name(), ) @@ -61,11 +61,11 @@ func (f *HandlerFactory) GetHandler(taskType TaskType) (TaskHandler, error) { } // CreateDefaultHandlers registers all default task handlers -func (f *HandlerFactory) CreateDefaultHandlers() { - f.RegisterHandler(TypeTopologyAnalysis, &TopologyAnalysisHandler{}) - f.RegisterHandler(TypeEventAnalysis, &EventAnalysisHandler{}) - f.RegisterHandler(TypeBatchImport, &BatchImportHandler{}) - f.RegisterHandler(TaskType(TaskTypeTest), NewTestTaskHandler()) +func (f *HandlerFactory) CreateDefaultHandlers(ctx context.Context) { + f.RegisterHandler(ctx, TypeTopologyAnalysis, &TopologyAnalysisHandler{}) + f.RegisterHandler(ctx, TypeEventAnalysis, &EventAnalysisHandler{}) + f.RegisterHandler(ctx, TypeBatchImport, &BatchImportHandler{}) + f.RegisterHandler(ctx, TaskType(TaskTypeTest), NewTestTaskHandler()) } // BaseHandler provides common functionality for all task handlers @@ -235,14 +235,14 @@ func (h *CompositeHandler) Name() string { } // DefaultHandlerFactory returns a HandlerFactory with all default handlers registered -func DefaultHandlerFactory() *HandlerFactory { +func DefaultHandlerFactory(ctx context.Context) *HandlerFactory { factory := NewHandlerFactory() - factory.CreateDefaultHandlers() + factory.CreateDefaultHandlers(ctx) return factory } // DefaultCompositeHandler returns a CompositeHandler with all default handlers -func DefaultCompositeHandler() TaskHandler { - factory := DefaultHandlerFactory() +func DefaultCompositeHandler(ctx context.Context) TaskHandler { + factory := DefaultHandlerFactory(ctx) return NewCompositeHandler(factory) } \ No newline at end of file diff --git a/task/initializer.go b/task/initializer.go index de75cea..1122a05 100644 --- a/task/initializer.go +++ b/task/initializer.go @@ -23,8 +23,8 @@ func InitTaskWorker(ctx context.Context, config config.ModelRTConfig, db *gorm.D // Create task handler factory handlerFactory := NewHandlerFactory() - handlerFactory.CreateDefaultHandlers() - handler := DefaultCompositeHandler() + handlerFactory.CreateDefaultHandlers(ctx) + handler := DefaultCompositeHandler(ctx) // Create task worker worker, err := NewTaskWorker(ctx, workerCfg, db, config.RabbitMQConfig, handler) @@ -38,4 +38,4 @@ func InitTaskWorker(ctx context.Context, config config.ModelRTConfig, db *gorm.D ) return worker, nil -} \ No newline at end of file +} diff --git a/task/queue_message.go b/task/queue_message.go index 6ce8028..95b717a 100644 --- a/task/queue_message.go +++ b/task/queue_message.go @@ -14,6 +14,8 @@ type TaskQueueMessage struct { TaskID uuid.UUID `json:"task_id"` TaskType TaskType `json:"task_type"` Priority int `json:"priority,omitempty"` // Optional, defaults to constants.TaskPriorityDefault + TraceID string `json:"trace_id,omitempty"` // propagated from the originating HTTP request + SpanID string `json:"span_id,omitempty"` // spanID of the step that enqueued this message } // NewTaskQueueMessage creates a new TaskQueueMessage with default priority diff --git a/task/queue_producer.go b/task/queue_producer.go index 650a2bf..97e6da7 100644 --- a/task/queue_producer.go +++ b/task/queue_producer.go @@ -11,11 +11,19 @@ import ( "modelRT/constants" "modelRT/logger" "modelRT/mq" + "modelRT/util" "github.com/gofrs/uuid" amqp "github.com/rabbitmq/amqp091-go" ) +// TaskMsgChan buffers task messages to be published to RabbitMQ asynchronously +var TaskMsgChan chan *TaskQueueMessage + +func init() { + TaskMsgChan = make(chan *TaskQueueMessage, 10000) +} + // QueueProducer handles publishing tasks to RabbitMQ type QueueProducer struct { conn *amqp.Connection @@ -212,4 +220,39 @@ func (p *QueueProducer) GetQueueInfo() (*amqp.Queue, error) { // PurgeQueue removes all messages from the task queue func (p *QueueProducer) PurgeQueue() (int, error) { return p.ch.QueuePurge(constants.TaskQueueName, false) +} + +// PushTaskToRabbitMQ reads from taskChan and publishes to RabbitMQ. +// Must be run as a goroutine; blocks until ctx is cancelled or taskChan is closed. +func PushTaskToRabbitMQ(ctx context.Context, cfg config.RabbitMQConfig, taskChan chan *TaskQueueMessage) { + producer, err := NewQueueProducer(ctx, cfg) + if err != nil { + logger.Error(ctx, "init task queue producer failed", "error", err) + return + } + defer producer.Close() + + for { + select { + case <-ctx.Done(): + logger.Info(ctx, "push task to RabbitMQ stopped by context cancel") + return + case msg, ok := <-taskChan: + if !ok { + logger.Info(ctx, "task channel closed, exiting push loop") + return + } + traceID := msg.TraceID + if traceID == "" { + traceID = msg.TaskID.String() // fallback when no HTTP trace was propagated + } + taskCtx := context.WithValue(ctx, constants.CtxKeyTraceID, traceID) + taskCtx = context.WithValue(taskCtx, constants.CtxKeySpanID, util.GenerateSpanID("task-publish")) + taskCtx = context.WithValue(taskCtx, constants.CtxKeyParentSpanID, msg.SpanID) + if err := producer.PublishTaskWithRetry(taskCtx, msg.TaskID, msg.TaskType, msg.Priority, 3); err != nil { + logger.Error(taskCtx, "publish task to RabbitMQ failed", + "task_id", msg.TaskID, "error", err) + } + } + } } \ No newline at end of file diff --git a/task/worker.go b/task/worker.go index 03f3728..6a47b56 100644 --- a/task/worker.go +++ b/task/worker.go @@ -14,6 +14,7 @@ import ( "modelRT/logger" "modelRT/mq" "modelRT/orm" + "modelRT/util" "github.com/gofrs/uuid" "github.com/panjf2000/ants/v2" @@ -282,6 +283,16 @@ func (w *TaskWorker) handleMessage(msg amqp.Delivery) { return } + // derive a per-task context carrying the trace propagated from the originating HTTP request + traceID := taskMsg.TraceID + if traceID == "" { + traceID = taskMsg.TaskID.String() // fallback when message carries no trace + } + taskCtx := context.WithValue(ctx, constants.CtxKeyTraceID, traceID) + taskCtx = context.WithValue(taskCtx, constants.CtxKeySpanID, util.GenerateSpanID("task-worker")) + taskCtx = context.WithValue(taskCtx, constants.CtxKeyParentSpanID, taskMsg.SpanID) + ctx = taskCtx + logger.Info(ctx, "Processing task", "task_id", taskMsg.TaskID, "task_type", taskMsg.TaskType, From 1b1f43db7fcf8958fd71dafaf42b4143ce88e0d9 Mon Sep 17 00:00:00 2001 From: douxu Date: Fri, 24 Apr 2026 17:14:46 +0800 Subject: [PATCH 31/43] feat: implement topology analysis async task with BFS connectivity check - add TopologyAnalysisHandler.Execute() with 5-phase BFS reachability check between start/end component UUIDs; support CheckInService flag to skip out-of-service nodes during traversal - carry task params through RabbitMQ message (TaskQueueMessage.Params) instead of re-querying DB in handler; update TaskHandler.Execute interface and all handler signatures accordingly - fix BuildMultiBranchTree UUIDFrom condition bug; return nodeMap for O(1) lookup; add QueryTopologicByStartUUID for directed traversal - add QueryBayByUUID/QueryBaysByUUIDs and QueryComponentsInServiceByUUIDs (two-column select) to database layer - add diagram.FindPath via LCA algorithm for tree path reconstruction - move initTracerProvider to middleware.InitTracerProvider; add OtelConfig struct to ModelRTConfig for endpoint configuration - update topology analysis params to start/end_component_uuid + check_in_service; remove dead topology init code --- config/config.go | 9 +- database/create_component.go | 2 +- database/create_measurement.go | 2 +- database/query_bay.go | 56 +++ database/query_component.go | 33 ++ database/query_topologic.go | 96 ++--- database/update_component.go | 2 +- deploy/jaeger.yaml | 60 +++ diagram/anchor_set.go | 3 +- diagram/component_set.go | 3 +- diagram/hash_test.go | 2 +- diagram/multi_branch_tree.go | 61 +++ diagram/topologic_set.go | 3 +- docs/docs.go | 485 +++++++++++++++++++++- docs/swagger.json | 485 +++++++++++++++++++++- docs/swagger.yaml | 319 +++++++++++++- go.mod | 37 +- go.sum | 85 +++- handler/async_task_create_handler.go | 25 +- logger/logger.go | 32 +- main.go | 24 +- middleware/trace.go | 96 ++++- model/redis_recommend.go | 1 - network/async_task_request.go | 5 +- network/circuit_diagram_update_request.go | 2 +- orm/async_motor.go | 1 - orm/busbar_section.go | 1 - orm/circuit_diagram_bay.go | 2 +- orm/circuit_diagram_component.go | 2 +- orm/circuit_diagram_grid.go | 2 +- orm/circuit_diagram_measurement.go | 2 +- orm/circuit_diagram_page.go | 2 +- orm/circuit_diagram_station.go | 2 +- orm/circuit_diagram_topologic.go | 2 +- orm/circuit_diagram_zone.go | 2 +- orm/demo.go | 3 +- task/handler_factory.go | 222 +++++++++- task/queue_message.go | 10 +- task/queue_producer.go | 24 +- task/test_task.go | 16 +- task/worker.go | 24 +- 41 files changed, 1962 insertions(+), 283 deletions(-) create mode 100644 database/query_bay.go create mode 100644 deploy/jaeger.yaml diff --git a/config/config.go b/config/config.go index 1425f45..c4a69d7 100644 --- a/config/config.go +++ b/config/config.go @@ -92,6 +92,12 @@ type DataRTConfig struct { Method string `mapstructure:"polling_api_method"` } +// OtelConfig define config struct of OpenTelemetry tracing +type OtelConfig struct { + Endpoint string `mapstructure:"endpoint"` // e.g. "localhost:4318" + Insecure bool `mapstructure:"insecure"` +} + // AsyncTaskConfig define config struct of asynchronous task system type AsyncTaskConfig struct { WorkerPoolSize int `mapstructure:"worker_pool_size"` @@ -115,7 +121,8 @@ type ModelRTConfig struct { LockerRedisConfig RedisConfig `mapstructure:"locker_redis"` StorageRedisConfig RedisConfig `mapstructure:"storage_redis"` AsyncTaskConfig AsyncTaskConfig `mapstructure:"async_task"` - PostgresDBURI string `mapstructure:"-"` + OtelConfig OtelConfig `mapstructure:"otel"` + PostgresDBURI string `mapstructure:"-"` } // ReadAndInitConfig return modelRT project config struct diff --git a/database/create_component.go b/database/create_component.go index 1c288c0..3d304a5 100644 --- a/database/create_component.go +++ b/database/create_component.go @@ -33,7 +33,7 @@ func CreateComponentIntoDB(ctx context.Context, tx *gorm.DB, componentInfo netwo Name: componentInfo.Name, Context: componentInfo.Context, Op: componentInfo.Op, - Ts: time.Now(), + TS: time.Now(), } result := tx.WithContext(cancelCtx).Create(&component) diff --git a/database/create_measurement.go b/database/create_measurement.go index 4085c94..30d6ad6 100644 --- a/database/create_measurement.go +++ b/database/create_measurement.go @@ -35,7 +35,7 @@ func CreateMeasurement(ctx context.Context, tx *gorm.DB, measurementInfo network BayUUID: globalUUID, ComponentUUID: globalUUID, Op: -1, - Ts: time.Now(), + TS: time.Now(), } result := tx.WithContext(cancelCtx).Create(&measurement) diff --git a/database/query_bay.go b/database/query_bay.go new file mode 100644 index 0000000..04ca639 --- /dev/null +++ b/database/query_bay.go @@ -0,0 +1,56 @@ +// Package database define database operation functions +package database + +import ( + "context" + "time" + + "modelRT/logger" + "modelRT/orm" + + "github.com/gofrs/uuid" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// QueryBayByUUID returns the Bay record matching bayUUID. +func QueryBayByUUID(ctx context.Context, tx *gorm.DB, bayUUID uuid.UUID) (*orm.Bay, error) { + var bay orm.Bay + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("bay_uuid = ?", bayUUID). + Clauses(clause.Locking{Strength: "UPDATE"}). + First(&bay) + + if result.Error != nil { + return nil, result.Error + } + return &bay, nil +} + +// QueryBaysByUUIDs returns Bay records matching the given UUIDs in a single query. +// The returned slice preserves database order; unmatched UUIDs are silently omitted. +func QueryBaysByUUIDs(ctx context.Context, tx *gorm.DB, bayUUIDs []uuid.UUID) ([]orm.Bay, error) { + if len(bayUUIDs) == 0 { + return nil, nil + } + + var bays []orm.Bay + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("bay_uuid IN ?", bayUUIDs). + Clauses(clause.Locking{Strength: "UPDATE"}). + Find(&bays) + + if result.Error != nil { + logger.Error(ctx, "query bays by uuids failed", "error", result.Error) + return nil, result.Error + } + return bays, nil +} diff --git a/database/query_component.go b/database/query_component.go index 9e8798d..73ca27c 100644 --- a/database/query_component.go +++ b/database/query_component.go @@ -148,6 +148,39 @@ func QueryLongIdentModelInfoByToken(ctx context.Context, tx *gorm.DB, measTag st return &resultComp, &meauserment, nil } +// QueryComponentsInServiceByUUIDs returns a map of global_uuid → in_service for the +// given UUIDs. Only global_uuid and in_service columns are selected for efficiency. +func QueryComponentsInServiceByUUIDs(ctx context.Context, tx *gorm.DB, uuids []uuid.UUID) (map[uuid.UUID]bool, error) { + if len(uuids) == 0 { + return make(map[uuid.UUID]bool), nil + } + + type row struct { + GlobalUUID uuid.UUID `gorm:"column:global_uuid"` + InService bool `gorm:"column:in_service"` + } + + var rows []row + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.Component{}). + Select("global_uuid, in_service"). + Where("global_uuid IN ?", uuids). + Scan(&rows) + if result.Error != nil { + return nil, result.Error + } + + m := make(map[uuid.UUID]bool, len(rows)) + for _, r := range rows { + m[r.GlobalUUID] = r.InService + } + return m, nil +} + // QueryShortIdentModelInfoByToken define func to query short identity model info by short token func QueryShortIdentModelInfoByToken(ctx context.Context, tx *gorm.DB, measTag string, condition *orm.Component) (*orm.Component, *orm.Measurement, error) { var resultComp orm.Component diff --git a/database/query_topologic.go b/database/query_topologic.go index eea21b1..4799875 100644 --- a/database/query_topologic.go +++ b/database/query_topologic.go @@ -32,71 +32,51 @@ func QueryTopologic(ctx context.Context, tx *gorm.DB) ([]orm.Topologic, error) { return topologics, nil } -// QueryTopologicFromDB return the result of query topologic info from DB -func QueryTopologicFromDB(ctx context.Context, tx *gorm.DB) (*diagram.MultiBranchTreeNode, error) { +// QueryTopologicByStartUUID returns all edges reachable from startUUID following +// directed uuid_from → uuid_to edges in the topologic table. +func QueryTopologicByStartUUID(ctx context.Context, tx *gorm.DB, startUUID uuid.UUID) ([]orm.Topologic, error) { + var topologics []orm.Topologic + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Clauses(clause.Locking{Strength: "UPDATE"}). + Raw(sql.RecursiveSQL, startUUID). + Scan(&topologics) + if result.Error != nil { + logger.Error(ctx, "query topologic by start uuid failed", "start_uuid", startUUID, "error", result.Error) + return nil, result.Error + } + return topologics, nil +} + +// QueryTopologicFromDB return the result of query topologic info from DB. +// Returns the root node and a flat nodeMap for O(1) lookup by UUID. +func QueryTopologicFromDB(ctx context.Context, tx *gorm.DB) (*diagram.MultiBranchTreeNode, map[uuid.UUID]*diagram.MultiBranchTreeNode, error) { topologicInfos, err := QueryTopologic(ctx, tx) if err != nil { logger.Error(ctx, "query topologic info failed", "error", err) - return nil, err + return nil, nil, err } - tree, err := BuildMultiBranchTree(topologicInfos) + tree, nodeMap, err := BuildMultiBranchTree(topologicInfos) if err != nil { logger.Error(ctx, "init topologic failed", "error", err) - return nil, err + return nil, nil, err } - return tree, nil + return tree, nodeMap, nil } -// InitCircuitDiagramTopologic return circuit diagram topologic info from postgres -func InitCircuitDiagramTopologic(topologicNodes []orm.Topologic) error { - var rootVertex *diagram.MultiBranchTreeNode - for _, node := range topologicNodes { - if node.UUIDFrom == constants.UUIDNil { - rootVertex = diagram.NewMultiBranchTree(node.UUIDFrom) - break - } - } - - if rootVertex == nil { - return fmt.Errorf("root vertex is nil") - } - - for _, node := range topologicNodes { - if node.UUIDFrom == constants.UUIDNil { - nodeVertex := diagram.NewMultiBranchTree(node.UUIDTo) - rootVertex.AddChild(nodeVertex) - } - } - - node := rootVertex - for _, nodeVertex := range node.Children { - nextVertexs := make([]*diagram.MultiBranchTreeNode, 0) - nextVertexs = append(nextVertexs, nodeVertex) - } - return nil -} - -// TODO 电流互感器不单独划分间隔,以母线、浇筑母线、变压器为间隔原件 -func IntervalBoundaryDetermine(uuid uuid.UUID) bool { - diagram.GetComponentMap(uuid.String()) - // TODO 判断 component 的类型是否为间隔 - // TODO 0xA1B2C3D4,高四位表示可以成为间隔的compoent类型的值为FFFF,普通 component 类型的值为 0000。低四位中前二位表示component的一级类型，例如母线 PT、母联/母分、进线等，低四位中后二位表示一级类型中包含的具体类型，例如母线 PT中包含的电压互感器、隔离开关、接地开关、避雷器、带电显示器等。 - num := uint32(0xA1B2C3D4) // 八位16进制数 - high16 := uint16(num >> 16) - fmt.Printf("原始值: 0x%X\n", num) // 输出: 0xA1B2C3D4 - fmt.Printf("高十六位: 0x%X\n", high16) // 输出: 0xA1B2 - return true -} - -// BuildMultiBranchTree return the multi branch tree by topologic info and component type map -func BuildMultiBranchTree(topologics []orm.Topologic) (*diagram.MultiBranchTreeNode, error) { +// BuildMultiBranchTree return the multi branch tree by topologic info. +// Returns the root node and a flat nodeMap for O(1) lookup by UUID. +func BuildMultiBranchTree(topologics []orm.Topologic) (*diagram.MultiBranchTreeNode, map[uuid.UUID]*diagram.MultiBranchTreeNode, error) { nodeMap := make(map[uuid.UUID]*diagram.MultiBranchTreeNode, len(topologics)*2) for _, topo := range topologics { if _, exists := nodeMap[topo.UUIDFrom]; !exists { - // skip special uuid - if topo.UUIDTo != constants.UUIDNil { + // UUIDNil is the virtual root sentinel — skip creating a regular node for it + if topo.UUIDFrom != constants.UUIDNil { nodeMap[topo.UUIDFrom] = &diagram.MultiBranchTreeNode{ ID: topo.UUIDFrom, Children: make([]*diagram.MultiBranchTreeNode, 0), @@ -105,7 +85,6 @@ func BuildMultiBranchTree(topologics []orm.Topologic) (*diagram.MultiBranchTreeN } if _, exists := nodeMap[topo.UUIDTo]; !exists { - // skip special uuid if topo.UUIDTo != constants.UUIDNil { nodeMap[topo.UUIDTo] = &diagram.MultiBranchTreeNode{ ID: topo.UUIDTo, @@ -118,10 +97,13 @@ func BuildMultiBranchTree(topologics []orm.Topologic) (*diagram.MultiBranchTreeN for _, topo := range topologics { var parent *diagram.MultiBranchTreeNode if topo.UUIDFrom == constants.UUIDNil { - parent = &diagram.MultiBranchTreeNode{ - ID: constants.UUIDNil, + if _, exists := nodeMap[constants.UUIDNil]; !exists { + nodeMap[constants.UUIDNil] = &diagram.MultiBranchTreeNode{ + ID: constants.UUIDNil, + Children: make([]*diagram.MultiBranchTreeNode, 0), + } } - nodeMap[constants.UUIDNil] = parent + parent = nodeMap[constants.UUIDNil] } else { parent = nodeMap[topo.UUIDFrom] } @@ -141,7 +123,7 @@ func BuildMultiBranchTree(topologics []orm.Topologic) (*diagram.MultiBranchTreeN // return root vertex root, exists := nodeMap[constants.UUIDNil] if !exists { - return nil, fmt.Errorf("root node not found") + return nil, nil, fmt.Errorf("root node not found") } - return root, nil + return root, nodeMap, nil } diff --git a/database/update_component.go b/database/update_component.go index 7957bc8..e08eea9 100644 --- a/database/update_component.go +++ b/database/update_component.go @@ -43,7 +43,7 @@ func UpdateComponentIntoDB(ctx context.Context, tx *gorm.DB, componentInfo netwo Name: componentInfo.Name, Context: componentInfo.Context, Op: componentInfo.Op, - Ts: time.Now(), + TS: time.Now(), } result = tx.Model(&orm.Component{}).WithContext(cancelCtx).Where("GLOBAL_UUID = ?", component.GlobalUUID).Updates(&updateParams) diff --git a/deploy/jaeger.yaml b/deploy/jaeger.yaml new file mode 100644 index 0000000..8dac477 --- /dev/null +++ b/deploy/jaeger.yaml @@ -0,0 +1,60 @@ +apiVersion: v1 +kind: Service +metadata: + name: jaeger + labels: + app: jaeger +spec: + ports: + - name: ui + port: 16686 + targetPort: 16686 + nodePort: 31686 # Jaeger UI，浏览器访问 http://:31686 + - name: collector-http + port: 14268 + targetPort: 14268 + nodePort: 31268 # Jaeger 原生 HTTP collector（非 OTel） + - name: otlp-http + port: 4318 + targetPort: 4318 + nodePort: 31318 # OTLP HTTP，集群外使用 :31318 + - name: otlp-grpc + port: 4317 + targetPort: 4317 + nodePort: 31317 # OTLP gRPC，集群外使用 :31317 + selector: + app: jaeger + type: NodePort +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jaeger +spec: + replicas: 1 + selector: + matchLabels: + app: jaeger + template: + metadata: + labels: + app: jaeger + spec: + containers: + - name: jaeger + image: jaegertracing/all-in-one:1.56 + env: + - name: COLLECTOR_OTLP_ENABLED + value: "true" + ports: + - containerPort: 16686 # UI + - containerPort: 14268 # Jaeger Collector + - containerPort: 4317 # OTLP gRPC + - containerPort: 4318 # OTLP HTTP + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi diff --git a/diagram/anchor_set.go b/diagram/anchor_set.go index 94a83f4..a6447ca 100644 --- a/diagram/anchor_set.go +++ b/diagram/anchor_set.go @@ -1,3 +1,4 @@ +// Package diagram provide diagram data structure and operation package diagram import ( @@ -31,11 +32,9 @@ func UpdateAnchorValue(componentUUID string, anchorValue string) bool { // StoreAnchorValue define func of store anchor value with componentUUID and anchor name func StoreAnchorValue(componentUUID string, anchorValue string) { anchorValueOverview.Store(componentUUID, anchorValue) - return } // DeleteAnchorValue define func of delete anchor value with componentUUID func DeleteAnchorValue(componentUUID string) { anchorValueOverview.Delete(componentUUID) - return } diff --git a/diagram/component_set.go b/diagram/component_set.go index 7a7b6c5..da9bddf 100644 --- a/diagram/component_set.go +++ b/diagram/component_set.go @@ -1,3 +1,4 @@ +// Package diagram provide diagram data structure and operation package diagram import ( @@ -33,11 +34,9 @@ func UpdateComponentMap(componentID int64, componentInfo *orm.Component) bool { // StoreComponentMap define func of store circuit diagram data with component uuid and component info func StoreComponentMap(componentUUID string, componentInfo *orm.Component) { diagramsOverview.Store(componentUUID, componentInfo) - return } // DeleteComponentMap define func of delete circuit diagram data with component uuid func DeleteComponentMap(componentUUID string) { diagramsOverview.Delete(componentUUID) - return } diff --git a/diagram/hash_test.go b/diagram/hash_test.go index ed320f3..5ba91fd 100644 --- a/diagram/hash_test.go +++ b/diagram/hash_test.go @@ -1,3 +1,4 @@ +// Package diagram provide diagram data structure and operation package diagram import ( @@ -29,5 +30,4 @@ func TestHMSet(t *testing.T) { fmt.Printf("err:%v\n", err) } fmt.Printf("res:%v\n", res) - return } diff --git a/diagram/multi_branch_tree.go b/diagram/multi_branch_tree.go index ceb6cfa..d88393b 100644 --- a/diagram/multi_branch_tree.go +++ b/diagram/multi_branch_tree.go @@ -1,3 +1,4 @@ +// Package diagram provide diagram data structure and operation package diagram import ( @@ -62,3 +63,63 @@ func (n *MultiBranchTreeNode) PrintTree(level int) { child.PrintTree(level + 1) } } + +// FindPath returns the ordered node sequence from startID to endID using the +// supplied nodeMap for O(1) lookup. It walks each node up to the root to find +// the LCA, then stitches the two half-paths together. +// Returns nil when either node is absent from nodeMap or no path exists. +func FindPath(startID, endID uuid.UUID, nodeMap map[uuid.UUID]*MultiBranchTreeNode) []*MultiBranchTreeNode { + startNode, ok := nodeMap[startID] + if !ok { + return nil + } + endNode, ok := nodeMap[endID] + if !ok { + return nil + } + + // collect ancestors (inclusive) from a node up to the root sentinel + ancestors := func(n *MultiBranchTreeNode) []*MultiBranchTreeNode { + var chain []*MultiBranchTreeNode + for n != nil { + chain = append(chain, n) + n = n.Parent + } + return chain + } + + startChain := ancestors(startNode) // [start, ..., root] + endChain := ancestors(endNode) // [end, ..., root] + + // index startChain by ID for fast LCA detection + startIdx := make(map[uuid.UUID]int, len(startChain)) + for i, node := range startChain { + startIdx[node.ID] = i + } + + // find LCA: first node in endChain that also appears in startChain + lcaEndPos := -1 + lcaStartPos := -1 + for i, node := range endChain { + if j, found := startIdx[node.ID]; found { + lcaEndPos = i + lcaStartPos = j + break + } + } + + if lcaEndPos < 0 { + return nil // disconnected + } + + // path = startChain[0..lcaStartPos] reversed + endChain[lcaEndPos..0] reversed + path := make([]*MultiBranchTreeNode, 0, lcaStartPos+lcaEndPos+1) + for i := 0; i <= lcaStartPos; i++ { + path = append(path, startChain[i]) + } + // append end-side (skip LCA to avoid duplication), reversed + for i := lcaEndPos - 1; i >= 0; i-- { + path = append(path, endChain[i]) + } + return path +} diff --git a/diagram/topologic_set.go b/diagram/topologic_set.go index 9dbbec3..607584f 100644 --- a/diagram/topologic_set.go +++ b/diagram/topologic_set.go @@ -1,3 +1,4 @@ +// Package diagram provide diagram data structure and operation package diagram import ( @@ -39,11 +40,9 @@ func UpdateGrapMap(pageID int64, graphInfo *Graph) bool { // StoreGraphMap define func of store circuit diagram topologic data with pageID and topologic info func StoreGraphMap(pageID int64, graphInfo *Graph) { graphOverview.Store(pageID, graphInfo) - return } // DeleteGraphMap define func of delete circuit diagram topologic data with pageID func DeleteGraphMap(pageID int64) { graphOverview.Delete(pageID) - return } diff --git a/docs/docs.go b/docs/docs.go index ab3d020..f674f98 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -102,13 +102,12 @@ const docTemplate = `{ "summary": "测量点推荐（搜索框自动补全）", "parameters": [ { - "description": "查询输入参数，例如 'trans' 或 'transformfeeder1_220.'", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/network.MeasurementRecommendRequest" - } + "type": "string", + "example": "\"grid1\"", + "description": "推荐关键词,例如 'grid1' 或 'grid1.'", + "name": "input", + "in": "query", + "required": true } ], "responses": { @@ -176,19 +175,400 @@ const docTemplate = `{ } } } + }, + "/monitors/data/realtime/stream/:clientID": { + "get": { + "description": "根据用户输入的clientID拉取对应的实时数据", + "tags": [ + "RealTime Component Websocket" + ], + "summary": "实时数据拉取 websocket api", + "responses": {} + } + }, + "/monitors/data/subscriptions": { + "post": { + "description": "根据用户输入的组件token,从 modelRT 服务中开始或结束对于量测节点的实时数据的订阅", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "RealTime Component" + ], + "summary": "开始或结束订阅实时数据", + "parameters": [ + { + "description": "量测节点实时数据订阅", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/network.RealTimeSubRequest" + } + } + ], + "responses": { + "2000": { + "description": "订阅实时数据结果列表", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.RealTimeSubPayload" + } + } + } + ] + } + }, + "3000": { + "description": "订阅实时数据结果列表", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.FailureResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.RealTimeSubPayload" + } + } + } + ] + } + } + } + } + }, + "/task/async": { + "post": { + "description": "创建新的异步任务并返回任务ID，任务将被提交到队列等待处理", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "创建异步任务", + "parameters": [ + { + "description": "任务创建请求", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/network.AsyncTaskCreateRequest" + } + } + ], + "responses": { + "200": { + "description": "任务创建成功", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.AsyncTaskCreateResponse" + } + } + } + ] + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } + }, + "/task/async/results": { + "get": { + "description": "根据任务ID列表查询异步任务的状态和结果", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "查询异步任务结果", + "parameters": [ + { + "type": "string", + "description": "任务ID列表，用逗号分隔", + "name": "task_ids", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "查询成功", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.AsyncTaskResultQueryResponse" + } + } + } + ] + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } + }, + "/task/async/{task_id}": { + "get": { + "description": "根据任务ID查询异步任务的详细状态和结果", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "查询异步任务详情", + "parameters": [ + { + "type": "string", + "description": "任务ID", + "name": "task_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "查询成功", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.AsyncTaskResult" + } + } + } + ] + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "404": { + "description": "任务不存在", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } + }, + "/task/async/{task_id}/cancel": { + "post": { + "description": "取消指定ID的异步任务（如果任务尚未开始执行）", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "取消异步任务", + "parameters": [ + { + "type": "string", + "description": "任务ID", + "name": "task_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "任务取消成功", + "schema": { + "$ref": "#/definitions/network.SuccessResponse" + } + }, + "400": { + "description": "请求参数错误或任务无法取消", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "404": { + "description": "任务不存在", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } } }, "definitions": { + "network.AsyncTaskCreateRequest": { + "type": "object", + "properties": { + "params": { + "description": "required: true", + "type": "object" + }, + "task_type": { + "description": "required: true\nenum: TOPOLOGY_ANALYSIS, PERFORMANCE_ANALYSIS, EVENT_ANALYSIS, BATCH_IMPORT", + "type": "string", + "example": "TOPOLOGY_ANALYSIS" + } + } + }, + "network.AsyncTaskCreateResponse": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "example": "123e4567-e89b-12d3-a456-426614174000" + } + } + }, + "network.AsyncTaskResult": { + "type": "object", + "properties": { + "created_at": { + "type": "integer", + "example": 1741846200 + }, + "error_code": { + "type": "integer", + "example": 400102 + }, + "error_detail": { + "type": "object" + }, + "error_message": { + "type": "string", + "example": "Component UUID not found" + }, + "finished_at": { + "type": "integer", + "example": 1741846205 + }, + "progress": { + "type": "integer", + "example": 65 + }, + "result": { + "type": "object" + }, + "status": { + "type": "string", + "example": "COMPLETED" + }, + "task_id": { + "type": "string", + "example": "123e4567-e89b-12d3-a456-426614174000" + }, + "task_type": { + "type": "string", + "example": "TOPOLOGY_ANALYSIS" + } + } + }, + "network.AsyncTaskResultQueryResponse": { + "type": "object", + "properties": { + "tasks": { + "type": "array", + "items": { + "$ref": "#/definitions/network.AsyncTaskResult" + } + }, + "total": { + "type": "integer", + "example": 3 + } + } + }, "network.FailureResponse": { "type": "object", "properties": { "code": { "type": "integer", - "example": 500 + "example": 3000 }, "msg": { "type": "string", - "example": "failed to get recommend data from redis" + "example": "process completed with partial failures" }, "payload": { "type": "object" @@ -216,15 +596,10 @@ const docTemplate = `{ " \"I_B_rms\"", "\"I_C_rms\"]" ] - } - } - }, - "network.MeasurementRecommendRequest": { - "type": "object", - "properties": { - "input": { + }, + "recommended_type": { "type": "string", - "example": "trans" + "example": "grid_tag" } } }, @@ -237,21 +612,93 @@ const docTemplate = `{ } } }, + "network.RealTimeMeasurementItem": { + "type": "object", + "properties": { + "interval": { + "type": "string", + "example": "1" + }, + "targets": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "[\"grid1.zone1.station1.ns1.tag1.bay.I11_A_rms\"", + "\"grid1.zone1.station1.ns1.tag1.tag1.bay.I11_B_rms\"]" + ] + } + } + }, + "network.RealTimeSubPayload": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "example": "5d72f2d9-e33a-4f1b-9c76-88a44b9a953e" + }, + "targets": { + "type": "array", + "items": { + "$ref": "#/definitions/network.TargetResult" + } + } + } + }, + "network.RealTimeSubRequest": { + "type": "object", + "properties": { + "action": { + "description": "required: true\nenum: [start, stop]", + "type": "string", + "example": "start" + }, + "client_id": { + "type": "string", + "example": "5d72f2d9-e33a-4f1b-9c76-88a44b9a953e" + }, + "measurements": { + "description": "required: true", + "type": "array", + "items": { + "$ref": "#/definitions/network.RealTimeMeasurementItem" + } + } + } + }, "network.SuccessResponse": { "type": "object", "properties": { "code": { "type": "integer", - "example": 200 + "example": 2000 }, "msg": { "type": "string", - "example": "success" + "example": "process completed" }, "payload": { "type": "object" } } + }, + "network.TargetResult": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "example": 20000 + }, + "id": { + "type": "string", + "example": "grid1.zone1.station1.ns1.tag1.transformfeeder1_220.I_A_rms" + }, + "msg": { + "type": "string", + "example": "subscription success" + } + } } } }` diff --git a/docs/swagger.json b/docs/swagger.json index 92f20fa..c6bb033 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -96,13 +96,12 @@ "summary": "测量点推荐（搜索框自动补全）", "parameters": [ { - "description": "查询输入参数，例如 'trans' 或 'transformfeeder1_220.'", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/network.MeasurementRecommendRequest" - } + "type": "string", + "example": "\"grid1\"", + "description": "推荐关键词,例如 'grid1' 或 'grid1.'", + "name": "input", + "in": "query", + "required": true } ], "responses": { @@ -170,19 +169,400 @@ } } } + }, + "/monitors/data/realtime/stream/:clientID": { + "get": { + "description": "根据用户输入的clientID拉取对应的实时数据", + "tags": [ + "RealTime Component Websocket" + ], + "summary": "实时数据拉取 websocket api", + "responses": {} + } + }, + "/monitors/data/subscriptions": { + "post": { + "description": "根据用户输入的组件token,从 modelRT 服务中开始或结束对于量测节点的实时数据的订阅", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "RealTime Component" + ], + "summary": "开始或结束订阅实时数据", + "parameters": [ + { + "description": "量测节点实时数据订阅", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/network.RealTimeSubRequest" + } + } + ], + "responses": { + "2000": { + "description": "订阅实时数据结果列表", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.RealTimeSubPayload" + } + } + } + ] + } + }, + "3000": { + "description": "订阅实时数据结果列表", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.FailureResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.RealTimeSubPayload" + } + } + } + ] + } + } + } + } + }, + "/task/async": { + "post": { + "description": "创建新的异步任务并返回任务ID，任务将被提交到队列等待处理", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "创建异步任务", + "parameters": [ + { + "description": "任务创建请求", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/network.AsyncTaskCreateRequest" + } + } + ], + "responses": { + "200": { + "description": "任务创建成功", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.AsyncTaskCreateResponse" + } + } + } + ] + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } + }, + "/task/async/results": { + "get": { + "description": "根据任务ID列表查询异步任务的状态和结果", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "查询异步任务结果", + "parameters": [ + { + "type": "string", + "description": "任务ID列表，用逗号分隔", + "name": "task_ids", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "查询成功", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.AsyncTaskResultQueryResponse" + } + } + } + ] + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } + }, + "/task/async/{task_id}": { + "get": { + "description": "根据任务ID查询异步任务的详细状态和结果", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "查询异步任务详情", + "parameters": [ + { + "type": "string", + "description": "任务ID", + "name": "task_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "查询成功", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.AsyncTaskResult" + } + } + } + ] + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "404": { + "description": "任务不存在", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } + }, + "/task/async/{task_id}/cancel": { + "post": { + "description": "取消指定ID的异步任务（如果任务尚未开始执行）", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "取消异步任务", + "parameters": [ + { + "type": "string", + "description": "任务ID", + "name": "task_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "任务取消成功", + "schema": { + "$ref": "#/definitions/network.SuccessResponse" + } + }, + "400": { + "description": "请求参数错误或任务无法取消", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "404": { + "description": "任务不存在", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } } }, "definitions": { + "network.AsyncTaskCreateRequest": { + "type": "object", + "properties": { + "params": { + "description": "required: true", + "type": "object" + }, + "task_type": { + "description": "required: true\nenum: TOPOLOGY_ANALYSIS, PERFORMANCE_ANALYSIS, EVENT_ANALYSIS, BATCH_IMPORT", + "type": "string", + "example": "TOPOLOGY_ANALYSIS" + } + } + }, + "network.AsyncTaskCreateResponse": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "example": "123e4567-e89b-12d3-a456-426614174000" + } + } + }, + "network.AsyncTaskResult": { + "type": "object", + "properties": { + "created_at": { + "type": "integer", + "example": 1741846200 + }, + "error_code": { + "type": "integer", + "example": 400102 + }, + "error_detail": { + "type": "object" + }, + "error_message": { + "type": "string", + "example": "Component UUID not found" + }, + "finished_at": { + "type": "integer", + "example": 1741846205 + }, + "progress": { + "type": "integer", + "example": 65 + }, + "result": { + "type": "object" + }, + "status": { + "type": "string", + "example": "COMPLETED" + }, + "task_id": { + "type": "string", + "example": "123e4567-e89b-12d3-a456-426614174000" + }, + "task_type": { + "type": "string", + "example": "TOPOLOGY_ANALYSIS" + } + } + }, + "network.AsyncTaskResultQueryResponse": { + "type": "object", + "properties": { + "tasks": { + "type": "array", + "items": { + "$ref": "#/definitions/network.AsyncTaskResult" + } + }, + "total": { + "type": "integer", + "example": 3 + } + } + }, "network.FailureResponse": { "type": "object", "properties": { "code": { "type": "integer", - "example": 500 + "example": 3000 }, "msg": { "type": "string", - "example": "failed to get recommend data from redis" + "example": "process completed with partial failures" }, "payload": { "type": "object" @@ -210,15 +590,10 @@ " \"I_B_rms\"", "\"I_C_rms\"]" ] - } - } - }, - "network.MeasurementRecommendRequest": { - "type": "object", - "properties": { - "input": { + }, + "recommended_type": { "type": "string", - "example": "trans" + "example": "grid_tag" } } }, @@ -231,21 +606,93 @@ } } }, + "network.RealTimeMeasurementItem": { + "type": "object", + "properties": { + "interval": { + "type": "string", + "example": "1" + }, + "targets": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "[\"grid1.zone1.station1.ns1.tag1.bay.I11_A_rms\"", + "\"grid1.zone1.station1.ns1.tag1.tag1.bay.I11_B_rms\"]" + ] + } + } + }, + "network.RealTimeSubPayload": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "example": "5d72f2d9-e33a-4f1b-9c76-88a44b9a953e" + }, + "targets": { + "type": "array", + "items": { + "$ref": "#/definitions/network.TargetResult" + } + } + } + }, + "network.RealTimeSubRequest": { + "type": "object", + "properties": { + "action": { + "description": "required: true\nenum: [start, stop]", + "type": "string", + "example": "start" + }, + "client_id": { + "type": "string", + "example": "5d72f2d9-e33a-4f1b-9c76-88a44b9a953e" + }, + "measurements": { + "description": "required: true", + "type": "array", + "items": { + "$ref": "#/definitions/network.RealTimeMeasurementItem" + } + } + } + }, "network.SuccessResponse": { "type": "object", "properties": { "code": { "type": "integer", - "example": 200 + "example": 2000 }, "msg": { "type": "string", - "example": "success" + "example": "process completed" }, "payload": { "type": "object" } } + }, + "network.TargetResult": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "example": 20000 + }, + "id": { + "type": "string", + "example": "grid1.zone1.station1.ns1.tag1.transformfeeder1_220.I_A_rms" + }, + "msg": { + "type": "string", + "example": "subscription success" + } + } } } } \ No newline at end of file diff --git a/docs/swagger.yaml b/docs/swagger.yaml index e540f45..598c8d9 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1,12 +1,71 @@ basePath: /api/v1 definitions: + network.AsyncTaskCreateRequest: + properties: + params: + description: 'required: true' + type: object + task_type: + description: |- + required: true + enum: TOPOLOGY_ANALYSIS, PERFORMANCE_ANALYSIS, EVENT_ANALYSIS, BATCH_IMPORT + example: TOPOLOGY_ANALYSIS + type: string + type: object + network.AsyncTaskCreateResponse: + properties: + task_id: + example: 123e4567-e89b-12d3-a456-426614174000 + type: string + type: object + network.AsyncTaskResult: + properties: + created_at: + example: 1741846200 + type: integer + error_code: + example: 400102 + type: integer + error_detail: + type: object + error_message: + example: Component UUID not found + type: string + finished_at: + example: 1741846205 + type: integer + progress: + example: 65 + type: integer + result: + type: object + status: + example: COMPLETED + type: string + task_id: + example: 123e4567-e89b-12d3-a456-426614174000 + type: string + task_type: + example: TOPOLOGY_ANALYSIS + type: string + type: object + network.AsyncTaskResultQueryResponse: + properties: + tasks: + items: + $ref: '#/definitions/network.AsyncTaskResult' + type: array + total: + example: 3 + type: integer + type: object network.FailureResponse: properties: code: - example: 500 + example: 3000 type: integer msg: - example: failed to get recommend data from redis + example: process completed with partial failures type: string payload: type: object @@ -27,11 +86,8 @@ definitions: items: type: string type: array - type: object - network.MeasurementRecommendRequest: - properties: - input: - example: trans + recommended_type: + example: grid_tag type: string type: object network.RealTimeDataPayload: @@ -40,17 +96,69 @@ definitions: description: TODO 增加example tag type: object type: object + network.RealTimeMeasurementItem: + properties: + interval: + example: "1" + type: string + targets: + example: + - '["grid1.zone1.station1.ns1.tag1.bay.I11_A_rms"' + - '"grid1.zone1.station1.ns1.tag1.tag1.bay.I11_B_rms"]' + items: + type: string + type: array + type: object + network.RealTimeSubPayload: + properties: + client_id: + example: 5d72f2d9-e33a-4f1b-9c76-88a44b9a953e + type: string + targets: + items: + $ref: '#/definitions/network.TargetResult' + type: array + type: object + network.RealTimeSubRequest: + properties: + action: + description: |- + required: true + enum: [start, stop] + example: start + type: string + client_id: + example: 5d72f2d9-e33a-4f1b-9c76-88a44b9a953e + type: string + measurements: + description: 'required: true' + items: + $ref: '#/definitions/network.RealTimeMeasurementItem' + type: array + type: object network.SuccessResponse: properties: code: - example: 200 + example: 2000 type: integer msg: - example: success + example: process completed type: string payload: type: object type: object + network.TargetResult: + properties: + code: + example: 20000 + type: integer + id: + example: grid1.zone1.station1.ns1.tag1.transformfeeder1_220.I_A_rms + type: string + msg: + example: subscription success + type: string + type: object host: localhost:8080 info: contact: @@ -110,12 +218,12 @@ paths: - application/json description: 根据用户输入的字符串，从 Redis 中查询可能的测量点或结构路径，并提供推荐列表。 parameters: - - description: 查询输入参数，例如 'trans' 或 'transformfeeder1_220.' - in: body - name: request + - description: 推荐关键词,例如 'grid1' 或 'grid1.' + example: '"grid1"' + in: query + name: input required: true - schema: - $ref: '#/definitions/network.MeasurementRecommendRequest' + type: string produces: - application/json responses: @@ -160,4 +268,187 @@ paths: summary: load circuit diagram info tags: - load circuit_diagram + /monitors/data/realtime/stream/:clientID: + get: + description: 根据用户输入的clientID拉取对应的实时数据 + responses: {} + summary: 实时数据拉取 websocket api + tags: + - RealTime Component Websocket + /monitors/data/subscriptions: + post: + consumes: + - application/json + description: 根据用户输入的组件token,从 modelRT 服务中开始或结束对于量测节点的实时数据的订阅 + parameters: + - description: 量测节点实时数据订阅 + in: body + name: request + required: true + schema: + $ref: '#/definitions/network.RealTimeSubRequest' + produces: + - application/json + responses: + "2000": + description: 订阅实时数据结果列表 + schema: + allOf: + - $ref: '#/definitions/network.SuccessResponse' + - properties: + payload: + $ref: '#/definitions/network.RealTimeSubPayload' + type: object + "3000": + description: 订阅实时数据结果列表 + schema: + allOf: + - $ref: '#/definitions/network.FailureResponse' + - properties: + payload: + $ref: '#/definitions/network.RealTimeSubPayload' + type: object + summary: 开始或结束订阅实时数据 + tags: + - RealTime Component + /task/async: + post: + consumes: + - application/json + description: 创建新的异步任务并返回任务ID，任务将被提交到队列等待处理 + parameters: + - description: 任务创建请求 + in: body + name: request + required: true + schema: + $ref: '#/definitions/network.AsyncTaskCreateRequest' + produces: + - application/json + responses: + "200": + description: 任务创建成功 + schema: + allOf: + - $ref: '#/definitions/network.SuccessResponse' + - properties: + payload: + $ref: '#/definitions/network.AsyncTaskCreateResponse' + type: object + "400": + description: 请求参数错误 + schema: + $ref: '#/definitions/network.FailureResponse' + "500": + description: 服务器内部错误 + schema: + $ref: '#/definitions/network.FailureResponse' + summary: 创建异步任务 + tags: + - AsyncTask + /task/async/{task_id}: + get: + consumes: + - application/json + description: 根据任务ID查询异步任务的详细状态和结果 + parameters: + - description: 任务ID + in: path + name: task_id + required: true + type: string + produces: + - application/json + responses: + "200": + description: 查询成功 + schema: + allOf: + - $ref: '#/definitions/network.SuccessResponse' + - properties: + payload: + $ref: '#/definitions/network.AsyncTaskResult' + type: object + "400": + description: 请求参数错误 + schema: + $ref: '#/definitions/network.FailureResponse' + "404": + description: 任务不存在 + schema: + $ref: '#/definitions/network.FailureResponse' + "500": + description: 服务器内部错误 + schema: + $ref: '#/definitions/network.FailureResponse' + summary: 查询异步任务详情 + tags: + - AsyncTask + /task/async/{task_id}/cancel: + post: + consumes: + - application/json + description: 取消指定ID的异步任务（如果任务尚未开始执行） + parameters: + - description: 任务ID + in: path + name: task_id + required: true + type: string + produces: + - application/json + responses: + "200": + description: 任务取消成功 + schema: + $ref: '#/definitions/network.SuccessResponse' + "400": + description: 请求参数错误或任务无法取消 + schema: + $ref: '#/definitions/network.FailureResponse' + "404": + description: 任务不存在 + schema: + $ref: '#/definitions/network.FailureResponse' + "500": + description: 服务器内部错误 + schema: + $ref: '#/definitions/network.FailureResponse' + summary: 取消异步任务 + tags: + - AsyncTask + /task/async/results: + get: + consumes: + - application/json + description: 根据任务ID列表查询异步任务的状态和结果 + parameters: + - description: 任务ID列表，用逗号分隔 + in: query + name: task_ids + required: true + type: string + produces: + - application/json + responses: + "200": + description: 查询成功 + schema: + allOf: + - $ref: '#/definitions/network.SuccessResponse' + - properties: + payload: + $ref: '#/definitions/network.AsyncTaskResultQueryResponse' + type: object + "400": + description: 请求参数错误 + schema: + $ref: '#/definitions/network.FailureResponse' + "500": + description: 服务器内部错误 + schema: + $ref: '#/definitions/network.FailureResponse' + summary: 查询异步任务结果 + tags: + - AsyncTask swagger: "2.0" diff --git a/go.mod b/go.mod index 40347c5..a815516 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module modelRT -go 1.24 +go 1.25.0 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 @@ -17,11 +17,16 @@ require ( github.com/rabbitmq/amqp091-go v1.10.0 github.com/redis/go-redis/v9 v9.7.3 github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.4 github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 + go.opentelemetry.io/contrib/propagators/b3 v1.43.0 + go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 + go.opentelemetry.io/otel/sdk v1.43.0 + go.opentelemetry.io/otel/trace v1.43.0 go.uber.org/zap v1.27.0 gorm.io/driver/mysql v1.5.7 gorm.io/driver/postgres v1.5.9 @@ -33,13 +38,16 @@ require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/bytedance/sonic v1.13.3 // indirect github.com/bytedance/sonic/loader v0.2.4 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/spec v0.21.0 // indirect @@ -49,6 +57,8 @@ require ( github.com/go-playground/validator/v10 v10.26.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect @@ -76,16 +86,23 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/arch v0.18.0 // indirect - golang.org/x/crypto v0.39.0 // indirect + golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sync v0.15.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect - golang.org/x/tools v0.33.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect + golang.org/x/tools v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/grpc v1.80.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 3b5290c..97e2fcf 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,10 @@ github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= @@ -42,6 +44,11 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= @@ -64,13 +71,19 @@ github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -126,8 +139,8 @@ github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzuk github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -151,8 +164,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= @@ -168,6 +181,26 @@ github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2W github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/propagators/b3 v1.43.0 h1:CETqV3QLLPTy5yNrqyMr41VnAOOD4lsRved7n4QG00A= +go.opentelemetry.io/contrib/propagators/b3 v1.43.0/go.mod h1:Q4mCiCdziYzpNR0g+6UqVotAlCDZdzz6L8jwY4knOrw= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= @@ -178,24 +211,24 @@ golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -203,8 +236,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -212,16 +245,24 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= -golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/handler/async_task_create_handler.go b/handler/async_task_create_handler.go index 6bf8a72..592460b 100644 --- a/handler/async_task_create_handler.go +++ b/handler/async_task_create_handler.go @@ -10,6 +10,8 @@ import ( "modelRT/task" "github.com/gin-gonic/gin" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" ) // AsyncTaskCreateHandler handles creation of asynchronous tasks @@ -67,13 +69,11 @@ func AsyncTaskCreateHandler(c *gin.Context) { // enqueue task to channel for async publishing to RabbitMQ msg := task.NewTaskQueueMessageWithPriority(asyncTask.TaskID, task.TaskType(request.TaskType), 5) - // propagate HTTP request trace so the async chain stays on the same traceID - if v, _ := ctx.Value(constants.CtxKeyTraceID).(string); v != "" { - msg.TraceID = v - } - if v, _ := ctx.Value(constants.CtxKeySpanID).(string); v != "" { - msg.SpanID = v - } + // propagate the current OTel span context so the async chain stays on the same trace + carrier := make(map[string]string) + otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(carrier)) + msg.TraceCarrier = carrier + msg.Params = request.Params task.TaskMsgChan <- msg logger.Info(ctx, "task enqueued to channel", "task_id", asyncTask.TaskID, "queue", constants.TaskQueueName) @@ -102,13 +102,18 @@ func validateTaskParams(taskType string, params map[string]any) bool { } func validateTopologyAnalysisParams(params map[string]any) bool { - // Check required parameters for topology analysis - if startUUID, ok := params["start_uuid"]; !ok || startUUID == "" { + if v, ok := params["start_component_uuid"]; !ok || v == "" { return false } - if endUUID, ok := params["end_uuid"]; !ok || endUUID == "" { + if v, ok := params["end_component_uuid"]; !ok || v == "" { return false } + // check_in_service is optional; validate type when present + if v, exists := params["check_in_service"]; exists { + if _, isBool := v.(bool); !isBool { + return false + } + } return true } diff --git a/logger/logger.go b/logger/logger.go index 0d4317a..3b4175f 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -6,8 +6,7 @@ import ( "path" "runtime" - "modelRT/constants" - + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -22,9 +21,6 @@ type Logger interface { type logger struct { ctx context.Context - traceID string - spanID string - pSpanID string _logger *zap.Logger } @@ -56,10 +52,10 @@ func makeLogFields(ctx context.Context, kv ...any) []zap.Field { kv = append(kv, "unknown") } - traceID, _ := ctx.Value(constants.CtxKeyTraceID).(string) - spanID, _ := ctx.Value(constants.CtxKeySpanID).(string) - parentSpanID, _ := ctx.Value(constants.CtxKeyParentSpanID).(string) - kv = append(kv, "traceID", traceID, "spanID", spanID, "parentSpanID", parentSpanID) + spanCtx := trace.SpanFromContext(ctx).SpanContext() + traceID := spanCtx.TraceID().String() + spanID := spanCtx.SpanID().String() + kv = append(kv, "traceID", traceID, "spanID", spanID) funcName, file, line := getLoggerCallerInfo() kv = append(kv, "func", funcName, "file", file, "line", line) @@ -100,25 +96,11 @@ func getLoggerCallerInfo() (funcName, file string, line int) { return } -// New returns a logger bound to ctx. Trace fields (traceID, spanID, parentSpanID) -// are extracted from ctx using typed keys, and are included in every log entry. +// New returns a logger bound to ctx. Trace fields (traceID, spanID) are extracted +// from the OTel span stored in ctx and included in every log entry. func New(ctx context.Context) Logger { - var traceID, spanID, pSpanID string - if v, _ := ctx.Value(constants.CtxKeyTraceID).(string); v != "" { - traceID = v - } - if v, _ := ctx.Value(constants.CtxKeySpanID).(string); v != "" { - spanID = v - } - if v, _ := ctx.Value(constants.CtxKeyParentSpanID).(string); v != "" { - pSpanID = v - } - return &logger{ ctx: ctx, - traceID: traceID, - spanID: spanID, - pSpanID: pSpanID, _logger: GetLoggerInstance(), } } diff --git a/main.go b/main.go index 103eca9..4121a1f 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ import ( "modelRT/database" "modelRT/diagram" "modelRT/logger" + "modelRT/middleware" "modelRT/model" "modelRT/mq" "modelRT/pool" @@ -38,6 +39,7 @@ import ( "github.com/panjf2000/ants/v2" swaggerFiles "github.com/swaggo/files" ginSwagger "github.com/swaggo/gin-swagger" + "go.opentelemetry.io/otel" "gorm.io/gorm" ) @@ -73,9 +75,6 @@ var ( func main() { flag.Parse() - startupSpanID := util.GenerateSpanID("startup") - ctx := context.WithValue(context.Background(), constants.CtxKeyTraceID, startupSpanID) - ctx = context.WithValue(ctx, constants.CtxKeySpanID, startupSpanID) configPath := filepath.Join(*modelRTConfigDir, *modelRTConfigName+"."+*modelRTConfigType) if _, err := os.Stat(configPath); os.IsNotExist(err) { @@ -101,6 +100,22 @@ func main() { logger.InitLoggerInstance(modelRTConfig.LoggerConfig) defer logger.GetLoggerInstance().Sync() + // init OTel TracerProvider + tp, tpErr := middleware.InitTracerProvider(context.Background(), modelRTConfig) + if tpErr != nil { + log.Printf("warn: OTLP tracer init failed, tracing disabled: %v", tpErr) + } + if tp != nil { + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + tp.Shutdown(shutdownCtx) + }() + } + + ctx, startupSpan := otel.Tracer("modelRT/main").Start(context.Background(), "startup") + defer startupSpan.End() + hostName, err := os.Hostname() if err != nil { logger.Error(ctx, "get host name failed", "error", err) @@ -226,7 +241,7 @@ func main() { } go realtimedata.StartComputingRealTimeDataLimit(ctx, allMeasurement) - tree, err := database.QueryTopologicFromDB(ctx, tx) + tree, _, err := database.QueryTopologicFromDB(ctx, tx) if err != nil { logger.Error(ctx, "load topologic info from postgres failed", "error", err) panic(err) @@ -285,3 +300,4 @@ func main() { } } } + diff --git a/middleware/trace.go b/middleware/trace.go index 604a08c..9712de8 100644 --- a/middleware/trace.go +++ b/middleware/trace.go @@ -1,41 +1,93 @@ -// Package middleware define gin framework middlewares +// Package middleware defines gin framework middlewares and OTel tracing infrastructure. package middleware import ( "bytes" "context" + "fmt" "io" "strings" "time" + "modelRT/config" "modelRT/constants" "modelRT/logger" - "modelRT/util" "github.com/gin-gonic/gin" + "go.opentelemetry.io/contrib/propagators/b3" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + sdkresource "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + oteltrace "go.opentelemetry.io/otel/trace" ) -// StartTrace define func of set trace info from request header -func StartTrace() gin.HandlerFunc { - return func(c *gin.Context) { - traceID := c.Request.Header.Get(constants.HeaderTraceID) - parentSpanID := c.Request.Header.Get(constants.HeaderSpanID) - spanID := util.GenerateSpanID(c.Request.RemoteAddr) - // if traceId is empty, it means it is the origin of the link. Set it to the spanId of this time. The originating spanId is the root spanId. - if traceID == "" { - // traceId identifies the entire request link, and spanId identifies the different services in the link. - traceID = spanID - } - c.Set(constants.HeaderTraceID, traceID) - c.Set(constants.HeaderSpanID, spanID) - c.Set(constants.HeaderParentSpanID, parentSpanID) +// InitTracerProvider creates an OTLP TracerProvider and registers it as the global provider. +// It also registers the B3 propagator to stay compatible with existing B3 infrastructure. +// The caller is responsible for calling Shutdown on the returned provider during graceful shutdown. +func InitTracerProvider(ctx context.Context, cfg config.ModelRTConfig) (*sdktrace.TracerProvider, error) { + opts := []otlptracehttp.Option{ + otlptracehttp.WithEndpoint(cfg.OtelConfig.Endpoint), + } + if cfg.OtelConfig.Insecure { + opts = append(opts, otlptracehttp.WithInsecure()) + } - // also inject into request context so c.Request.Context() carries trace values - reqCtx := c.Request.Context() - reqCtx = context.WithValue(reqCtx, constants.CtxKeyTraceID, traceID) - reqCtx = context.WithValue(reqCtx, constants.CtxKeySpanID, spanID) - reqCtx = context.WithValue(reqCtx, constants.CtxKeyParentSpanID, parentSpanID) - c.Request = c.Request.WithContext(reqCtx) + exporter, err := otlptracehttp.New(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("create OTLP exporter: %w", err) + } + + res := sdkresource.NewSchemaless( + attribute.String("service.name", cfg.ServiceName), + attribute.String("deployment.environment", cfg.DeployEnv), + ) + + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + ) + + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(b3.New()) + + return tp, nil +} + +// StartTrace extracts upstream B3 trace context from request headers and starts a server span. +// Typed context keys are also injected for backward compatibility with the existing logger +// until the logger is migrated to read from the OTel span context (Step 6). +func StartTrace() gin.HandlerFunc { + tracer := otel.Tracer("modelRT/http") + return func(c *gin.Context) { + // Extract upstream trace context from B3 headers (X-B3-TraceId etc.) + ctx := otel.GetTextMapPropagator().Extract( + c.Request.Context(), + propagation.HeaderCarrier(c.Request.Header), + ) + + spanName := c.FullPath() + if spanName == "" { + spanName = c.Request.URL.Path + } + ctx, span := tracer.Start(ctx, spanName, + oteltrace.WithSpanKind(oteltrace.SpanKindServer), + ) + defer span.End() + + // backward compat: inject typed keys so existing logger reads work until Step 6 + spanCtx := span.SpanContext() + ctx = context.WithValue(ctx, constants.CtxKeyTraceID, spanCtx.TraceID().String()) + ctx = context.WithValue(ctx, constants.CtxKeySpanID, spanCtx.SpanID().String()) + + c.Request = c.Request.WithContext(ctx) + + // set in gin context for accessLog (logger.New(c) reads via gin.Context.Value) + c.Set(constants.HeaderTraceID, spanCtx.TraceID().String()) + c.Set(constants.HeaderSpanID, spanCtx.SpanID().String()) c.Next() } diff --git a/model/redis_recommend.go b/model/redis_recommend.go index e0ac507..ddfef2a 100644 --- a/model/redis_recommend.go +++ b/model/redis_recommend.go @@ -550,7 +550,6 @@ func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy co IsFuzzy: true, Err: nil, } - return } // runFuzzySearch define func to process redis fuzzy search diff --git a/network/async_task_request.go b/network/async_task_request.go index 9534b76..bf6a55a 100644 --- a/network/async_task_request.go +++ b/network/async_task_request.go @@ -62,8 +62,9 @@ type AsyncTaskStatusUpdate struct { // TopologyAnalysisParams defines the parameters for topology analysis task type TopologyAnalysisParams struct { - StartUUID string `json:"start_uuid" example:"comp-001" description:"起始元件UUID"` - EndUUID string `json:"end_uuid" example:"comp-999" description:"目标元件UUID"` + StartComponentUUID string `json:"start_component_uuid" example:"550e8400-e29b-41d4-a716-446655440000" description:"起始元件UUID"` + EndComponentUUID string `json:"end_component_uuid" example:"550e8400-e29b-41d4-a716-446655440001" description:"目标元件UUID"` + CheckInService bool `json:"check_in_service" example:"true" description:"是否检查路径上元件的投运状态，默认为true"` } // PerformanceAnalysisParams defines the parameters for performance analysis task diff --git a/network/circuit_diagram_update_request.go b/network/circuit_diagram_update_request.go index 86b1a76..76ed6c2 100644 --- a/network/circuit_diagram_update_request.go +++ b/network/circuit_diagram_update_request.go @@ -158,7 +158,7 @@ func ConvertComponentUpdateInfosToComponents(updateInfo ComponentUpdateInfo) (*o // Op: info.Op, // Tag: info.Tag, // 其他字段可根据需要补充 - Ts: time.Now(), + TS: time.Now(), } return component, nil } diff --git a/orm/async_motor.go b/orm/async_motor.go index 017e46e..48547ec 100644 --- a/orm/async_motor.go +++ b/orm/async_motor.go @@ -85,7 +85,6 @@ func (a *AsyncMotor) TableName() string { // SetComponentID func implement BasicModelInterface interface func (a *AsyncMotor) SetComponentID(componentID int64) { a.ComponentID = componentID - return } // ReturnTableName func implement BasicModelInterface interface diff --git a/orm/busbar_section.go b/orm/busbar_section.go index 764057f..d953d2f 100644 --- a/orm/busbar_section.go +++ b/orm/busbar_section.go @@ -72,7 +72,6 @@ func (b *BusbarSection) TableName() string { // SetComponentID func implement BasicModelInterface interface func (b *BusbarSection) SetComponentID(componentID int64) { b.ComponentID = componentID - return } // ReturnTableName func implement BasicModelInterface interface diff --git a/orm/circuit_diagram_bay.go b/orm/circuit_diagram_bay.go index 4554bc3..4396219 100644 --- a/orm/circuit_diagram_bay.go +++ b/orm/circuit_diagram_bay.go @@ -34,7 +34,7 @@ type Bay struct { DevEtc JSONMap `gorm:"column:dev_etc;type:jsonb;not null;default:'[]'"` Components []uuid.UUID `gorm:"column:components;type:uuid[];not null;default:'{}'"` Op int `gorm:"column:op;not null;default:-1"` - Ts time.Time `gorm:"column:ts;type:timestamptz;not null;default:CURRENT_TIMESTAMP"` + TS time.Time `gorm:"column:ts;type:timestamptz;not null;default:CURRENT_TIMESTAMP"` } // TableName func respresent return table name of Bay diff --git a/orm/circuit_diagram_component.go b/orm/circuit_diagram_component.go index df41bc5..a292c6a 100644 --- a/orm/circuit_diagram_component.go +++ b/orm/circuit_diagram_component.go @@ -27,7 +27,7 @@ type Component struct { Label JSONMap `gorm:"column:label;type:jsonb;not null;default:'{}'"` Context JSONMap `gorm:"column:context;type:jsonb;not null;default:'{}'"` Op int `gorm:"column:op;not null;default:-1"` - Ts time.Time `gorm:"column:ts;type:timestamptz;not null;default:current_timestamp;autoCreateTime"` + TS time.Time `gorm:"column:ts;type:timestamptz;not null;default:current_timestamp;autoCreateTime"` } // TableName func respresent return table name of Component diff --git a/orm/circuit_diagram_grid.go b/orm/circuit_diagram_grid.go index 43d90e5..86da8a8 100644 --- a/orm/circuit_diagram_grid.go +++ b/orm/circuit_diagram_grid.go @@ -12,7 +12,7 @@ type Grid struct { Name string `gorm:"column:name"` Description string `gorm:"column:description"` Op int `gorm:"column:op"` - Ts time.Time `gorm:"column:ts"` + TS time.Time `gorm:"column:ts"` } // TableName func respresent return table name of Grid diff --git a/orm/circuit_diagram_measurement.go b/orm/circuit_diagram_measurement.go index 4abc441..e281559 100644 --- a/orm/circuit_diagram_measurement.go +++ b/orm/circuit_diagram_measurement.go @@ -20,7 +20,7 @@ type Measurement struct { BayUUID uuid.UUID `gorm:"column:bay_uuid;type:uuid;not null"` ComponentUUID uuid.UUID `gorm:"column:component_uuid;type:uuid;not null"` Op int `gorm:"column:op;not null;default:-1"` - Ts time.Time `gorm:"column:ts;type:timestamptz;not null;default:CURRENT_TIMESTAMP"` + TS time.Time `gorm:"column:ts;type:timestamptz;not null;default:CURRENT_TIMESTAMP"` } // TableName func respresent return table name of Measurement diff --git a/orm/circuit_diagram_page.go b/orm/circuit_diagram_page.go index 172a693..2e3c50a 100644 --- a/orm/circuit_diagram_page.go +++ b/orm/circuit_diagram_page.go @@ -12,7 +12,7 @@ type Page struct { Context JSONMap `gorm:"column:context;type:jsonb;default:'{}'"` Description string `gorm:"column:description"` Op int `gorm:"column:op"` - Ts time.Time `gorm:"column:ts"` + TS time.Time `gorm:"column:ts"` } // TableName func respresent return table name of Page diff --git a/orm/circuit_diagram_station.go b/orm/circuit_diagram_station.go index e937257..5059e97 100644 --- a/orm/circuit_diagram_station.go +++ b/orm/circuit_diagram_station.go @@ -14,7 +14,7 @@ type Station struct { Description string `gorm:"column:description"` IsLocal bool `gorm:"column:is_local"` Op int `gorm:"column:op"` - Ts time.Time `gorm:"column:ts"` + TS time.Time `gorm:"column:ts"` } // TableName func respresent return table name of Station diff --git a/orm/circuit_diagram_topologic.go b/orm/circuit_diagram_topologic.go index d85137b..7f444e6 100644 --- a/orm/circuit_diagram_topologic.go +++ b/orm/circuit_diagram_topologic.go @@ -16,7 +16,7 @@ type Topologic struct { Flag int `gorm:"column:flag"` Description string `gorm:"column:description;size:512;not null;default:''"` Op int `gorm:"column:op;not null;default:-1"` - Ts time.Time `gorm:"column:ts;type:timestamptz;not null;default:CURRENT_TIMESTAMP"` + TS time.Time `gorm:"column:ts;type:timestamptz;not null;default:CURRENT_TIMESTAMP"` } // TableName func respresent return table name of Page diff --git a/orm/circuit_diagram_zone.go b/orm/circuit_diagram_zone.go index f50aaee..85caf39 100644 --- a/orm/circuit_diagram_zone.go +++ b/orm/circuit_diagram_zone.go @@ -13,7 +13,7 @@ type Zone struct { Name string `gorm:"column:name"` Description string `gorm:"column:description"` Op int `gorm:"column:op"` - Ts time.Time `gorm:"column:ts"` + TS time.Time `gorm:"column:ts"` } // TableName func respresent return table name of Zone diff --git a/orm/demo.go b/orm/demo.go index 8b0991b..2483066 100644 --- a/orm/demo.go +++ b/orm/demo.go @@ -14,7 +14,7 @@ type Demo struct { UIAlarm float32 `gorm:"column:ui_alarm" json:"ui_alarm"` // 低电流告警值 OIAlarm float32 `gorm:"column:oi_alarm" json:"oi_alarm"` // 高电流告警值 Op int `gorm:"column:op" json:"op"` // 操作人 ID - Ts time.Time `gorm:"column:ts" json:"ts"` // 操作时间 + TS time.Time `gorm:"column:ts" json:"ts"` // 操作时间 } // TableName func respresent return table name of busbar section @@ -25,7 +25,6 @@ func (d *Demo) TableName() string { // SetComponentID func implement BasicModelInterface interface func (d *Demo) SetComponentID(componentID int64) { d.ComponentID = componentID - return } // ReturnTableName func implement BasicModelInterface interface diff --git a/task/handler_factory.go b/task/handler_factory.go index 1efdf65..ae1d951 100644 --- a/task/handler_factory.go +++ b/task/handler_factory.go @@ -5,8 +5,11 @@ import ( "context" "fmt" "sync" + "time" + "modelRT/database" "modelRT/logger" + "modelRT/orm" "github.com/gofrs/uuid" "gorm.io/gorm" @@ -14,8 +17,8 @@ import ( // TaskHandler defines the interface for task processors type TaskHandler interface { - // Execute processes a task with the given ID and type - Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, db *gorm.DB) error + // Execute processes a task with the given ID, type, and params from the MQ message + Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, params map[string]any, db *gorm.DB) error // CanHandle returns true if this handler can process the given task type CanHandle(taskType TaskType) bool // Name returns the name of the handler for logging and metrics @@ -95,26 +98,205 @@ func NewTopologyAnalysisHandler() *TopologyAnalysisHandler { } } -// Execute processes a topology analysis task -func (h *TopologyAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, db *gorm.DB) error { - logger.Info(ctx, "Starting topology analysis", +// Execute processes a topology analysis task. +// Params (all sourced from the MQ message, no DB lookup needed): +// - start_component_uuid (string, required): BFS origin +// - end_component_uuid (string, required): reachability target +// - check_in_service (bool, optional, default true): skip out-of-service components +func (h *TopologyAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, params map[string]any, db *gorm.DB) error { + logger.Info(ctx, "topology analysis started", "task_id", taskID) + + // Phase 1: parse params from MQ message + startComponentUUID, endComponentUUID, checkInService, err := parseTopologyAnalysisParams(params) + if err != nil { + return fmt.Errorf("invalid topology analysis params: %w", err) + } + + logger.Info(ctx, "topology params parsed", "task_id", taskID, - "task_type", taskType, + "start", startComponentUUID, + "end", endComponentUUID, + "check_in_service", checkInService, ) - // TODO: Implement actual topology analysis logic - // This would typically involve: - // 1. Fetching task parameters from database - // 2. Performing topology analysis (checking for islands, shorts, etc.) - // 3. Storing results in database - // 4. Updating task status + if err := database.UpdateAsyncTaskProgress(ctx, db, taskID, 20); err != nil { + logger.Warn(ctx, "update progress failed", "task_id", taskID, "progress", 20, "error", err) + } - // Simulate work - logger.Info(ctx, "Topology analysis completed", + // Phase 2: query topology edges from startComponentUUID, build adjacency list + topoEdges, err := database.QueryTopologicByStartUUID(ctx, db, startComponentUUID) + if err != nil { + return fmt.Errorf("query topology from start node: %w", err) + } + + // adjacency list: uuid_from → []uuid_to + adjMap := make(map[uuid.UUID][]uuid.UUID, len(topoEdges)) + // collect all UUIDs for batch InService query + allUUIDs := make(map[uuid.UUID]struct{}, len(topoEdges)*2) + allUUIDs[startComponentUUID] = struct{}{} + for _, edge := range topoEdges { + adjMap[edge.UUIDFrom] = append(adjMap[edge.UUIDFrom], edge.UUIDTo) + allUUIDs[edge.UUIDFrom] = struct{}{} + allUUIDs[edge.UUIDTo] = struct{}{} + } + + if err := database.UpdateAsyncTaskProgress(ctx, db, taskID, 40); err != nil { + logger.Warn(ctx, "update progress failed", "task_id", taskID, "progress", 40, "error", err) + } + + // Phase 3: batch-load InService status (only when checkInService is true) + inServiceMap := make(map[uuid.UUID]bool) + if checkInService { + uuidSlice := make([]uuid.UUID, 0, len(allUUIDs)) + for id := range allUUIDs { + uuidSlice = append(uuidSlice, id) + } + inServiceMap, err = database.QueryComponentsInServiceByUUIDs(ctx, db, uuidSlice) + if err != nil { + return fmt.Errorf("query component in_service status: %w", err) + } + + // check the start node itself before BFS + if !inServiceMap[startComponentUUID] { + return persistTopologyResult(ctx, db, taskID, startComponentUUID, endComponentUUID, + checkInService, false, nil, &startComponentUUID) + } + } + + if err := database.UpdateAsyncTaskProgress(ctx, db, taskID, 60); err != nil { + logger.Warn(ctx, "update progress failed", "task_id", taskID, "progress", 60, "error", err) + } + + // Phase 4: BFS reachability check + visited := make(map[uuid.UUID]struct{}) + parent := make(map[uuid.UUID]uuid.UUID) // for path reconstruction + queue := []uuid.UUID{startComponentUUID} + visited[startComponentUUID] = struct{}{} + isReachable := false + var blockedBy *uuid.UUID + + for len(queue) > 0 { + cur := queue[0] + queue = queue[1:] + + if cur == endComponentUUID { + isReachable = true + break + } + + for _, next := range adjMap[cur] { + if _, seen := visited[next]; seen { + continue + } + if checkInService && !inServiceMap[next] { + // record first out-of-service blocker but keep searching other branches + if blockedBy == nil { + id := next + blockedBy = &id + } + continue + } + visited[next] = struct{}{} + parent[next] = cur + queue = append(queue, next) + } + } + + if err := database.UpdateAsyncTaskProgress(ctx, db, taskID, 80); err != nil { + logger.Warn(ctx, "update progress failed", "task_id", taskID, "progress", 80, "error", err) + } + + // Phase 5: reconstruct path (if reachable) and persist result + var path []uuid.UUID + if isReachable { + blockedBy = nil // reachable path found — clear any partial blocker + path = reconstructPath(parent, startComponentUUID, endComponentUUID) + } + + return persistTopologyResult(ctx, db, taskID, startComponentUUID, endComponentUUID, + checkInService, isReachable, path, blockedBy) +} + +// parseTopologyAnalysisParams extracts and validates the three required fields. +// check_in_service defaults to true when absent. +func parseTopologyAnalysisParams(params map[string]any) (startID, endID uuid.UUID, checkInService bool, err error) { + startStr, ok := params["start_component_uuid"].(string) + if !ok || startStr == "" { + err = fmt.Errorf("missing or invalid start_component_uuid") + return + } + endStr, ok := params["end_component_uuid"].(string) + if !ok || endStr == "" { + err = fmt.Errorf("missing or invalid end_component_uuid") + return + } + startID, err = uuid.FromString(startStr) + if err != nil { + err = fmt.Errorf("parse start_component_uuid %q: %w", startStr, err) + return + } + endID, err = uuid.FromString(endStr) + if err != nil { + err = fmt.Errorf("parse end_component_uuid %q: %w", endStr, err) + return + } + + // check_in_service defaults to true + checkInService = true + if v, exists := params["check_in_service"]; exists { + if b, isBool := v.(bool); isBool { + checkInService = b + } + } + return +} + +// reconstructPath walks the parent map backwards from end to start. +func reconstructPath(parent map[uuid.UUID]uuid.UUID, start, end uuid.UUID) []uuid.UUID { + var path []uuid.UUID + for cur := end; cur != start; cur = parent[cur] { + path = append(path, cur) + } + path = append(path, start) + // reverse: path was built end→start + for i, j := 0, len(path)-1; i < j; i, j = i+1, j-1 { + path[i], path[j] = path[j], path[i] + } + return path +} + +// persistTopologyResult serialises the analysis outcome and writes it to async_task_result. +func persistTopologyResult( + ctx context.Context, db *gorm.DB, taskID uuid.UUID, + startID, endID uuid.UUID, checkInService, isReachable bool, + path []uuid.UUID, blockedBy *uuid.UUID, +) error { + pathStrs := make([]string, 0, len(path)) + for _, id := range path { + pathStrs = append(pathStrs, id.String()) + } + + result := orm.JSONMap{ + "start_component_uuid": startID.String(), + "end_component_uuid": endID.String(), + "check_in_service": checkInService, + "is_reachable": isReachable, + "path": pathStrs, + "computed_at": time.Now().Unix(), + } + if blockedBy != nil { + result["blocked_by"] = blockedBy.String() + } + + if err := database.CreateAsyncTaskResult(ctx, db, taskID, result); err != nil { + return fmt.Errorf("save task result: %w", err) + } + + logger.Info(ctx, "topology analysis completed", "task_id", taskID, - "task_type", taskType, + "is_reachable", isReachable, + "path_length", len(path), ) - return nil } @@ -136,7 +318,7 @@ func NewEventAnalysisHandler() *EventAnalysisHandler { } // Execute processes an event analysis task -func (h *EventAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, db *gorm.DB) error { +func (h *EventAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, params map[string]any, db *gorm.DB) error { logger.Info(ctx, "Starting event analysis", "task_id", taskID, "task_type", taskType, @@ -176,7 +358,7 @@ func NewBatchImportHandler() *BatchImportHandler { } // Execute processes a batch import task -func (h *BatchImportHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, db *gorm.DB) error { +func (h *BatchImportHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, params map[string]any, db *gorm.DB) error { logger.Info(ctx, "Starting batch import", "task_id", taskID, "task_type", taskType, @@ -214,13 +396,13 @@ func NewCompositeHandler(factory *HandlerFactory) *CompositeHandler { } // Execute delegates task execution to the appropriate handler -func (h *CompositeHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, db *gorm.DB) error { +func (h *CompositeHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, params map[string]any, db *gorm.DB) error { handler, err := h.factory.GetHandler(taskType) if err != nil { return fmt.Errorf("failed to get handler for task type %s: %w", taskType, err) } - return handler.Execute(ctx, taskID, taskType, db) + return handler.Execute(ctx, taskID, taskType, params, db) } // CanHandle returns true if any registered handler can handle the task type diff --git a/task/queue_message.go b/task/queue_message.go index 95b717a..9cc5ebf 100644 --- a/task/queue_message.go +++ b/task/queue_message.go @@ -11,11 +11,11 @@ import ( // TaskQueueMessage defines minimal message structure for RabbitMQ/Redis queue dispatch // This struct is designed to be lightweight for efficient message transport type TaskQueueMessage struct { - TaskID uuid.UUID `json:"task_id"` - TaskType TaskType `json:"task_type"` - Priority int `json:"priority,omitempty"` // Optional, defaults to constants.TaskPriorityDefault - TraceID string `json:"trace_id,omitempty"` // propagated from the originating HTTP request - SpanID string `json:"span_id,omitempty"` // spanID of the step that enqueued this message + TaskID uuid.UUID `json:"task_id"` + TaskType TaskType `json:"task_type"` + Priority int `json:"priority,omitempty"` // Optional, defaults to constants.TaskPriorityDefault + TraceCarrier map[string]string `json:"trace_carrier,omitempty"` // OTel propagation carrier (B3 headers) + Params map[string]any `json:"params,omitempty"` // Task-specific parameters, set by the HTTP handler } // NewTaskQueueMessage creates a new TaskQueueMessage with default priority diff --git a/task/queue_producer.go b/task/queue_producer.go index 97e6da7..bcaaffc 100644 --- a/task/queue_producer.go +++ b/task/queue_producer.go @@ -11,10 +11,13 @@ import ( "modelRT/constants" "modelRT/logger" "modelRT/mq" - "modelRT/util" "github.com/gofrs/uuid" amqp "github.com/rabbitmq/amqp091-go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" + oteltrace "go.opentelemetry.io/otel/trace" ) // TaskMsgChan buffers task messages to be published to RabbitMQ asynchronously @@ -115,6 +118,11 @@ func (p *QueueProducer) PublishTask(ctx context.Context, taskID uuid.UUID, taskT return fmt.Errorf("invalid task message: taskID=%s, taskType=%s", taskID, taskType) } + // Inject OTel trace context so the consumer (worker) can restore the span chain + carrier := make(map[string]string) + otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(carrier)) + message.TraceCarrier = carrier + // Convert message to JSON body, err := json.Marshal(message) if err != nil { @@ -242,17 +250,17 @@ func PushTaskToRabbitMQ(ctx context.Context, cfg config.RabbitMQConfig, taskChan logger.Info(ctx, "task channel closed, exiting push loop") return } - traceID := msg.TraceID - if traceID == "" { - traceID = msg.TaskID.String() // fallback when no HTTP trace was propagated - } - taskCtx := context.WithValue(ctx, constants.CtxKeyTraceID, traceID) - taskCtx = context.WithValue(taskCtx, constants.CtxKeySpanID, util.GenerateSpanID("task-publish")) - taskCtx = context.WithValue(taskCtx, constants.CtxKeyParentSpanID, msg.SpanID) + // Restore trace context from the handler that enqueued this message + taskCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(msg.TraceCarrier)) + taskCtx, pubSpan := otel.Tracer("modelRT/task").Start(taskCtx, "task.publish", + oteltrace.WithAttributes(attribute.String("task_id", msg.TaskID.String())), + ) if err := producer.PublishTaskWithRetry(taskCtx, msg.TaskID, msg.TaskType, msg.Priority, 3); err != nil { + pubSpan.RecordError(err) logger.Error(taskCtx, "publish task to RabbitMQ failed", "task_id", msg.TaskID, "error", err) } + pubSpan.End() } } } \ No newline at end of file diff --git a/task/test_task.go b/task/test_task.go index 4b38ac9..d85b682 100644 --- a/task/test_task.go +++ b/task/test_task.go @@ -141,26 +141,20 @@ func NewTestTaskHandler() *TestTaskHandler { } // Execute processes a test task using the unified task interface -func (h *TestTaskHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, db *gorm.DB) error { +func (h *TestTaskHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, params map[string]any, db *gorm.DB) error { logger.Info(ctx, "Executing test task", "task_id", taskID, "task_type", taskType, ) - // Fetch task parameters from database - asyncTask, err := database.GetAsyncTaskByID(ctx, db, taskID) - if err != nil { - return fmt.Errorf("failed toser fetch task: %w", err) - } - - // Convert params map to TestTaskParams - params := &TestTaskParams{} - if err := params.FromMap(map[string]interface{}(asyncTask.Params)); err != nil { + // Convert params from MQ message to TestTaskParams + taskParams := &TestTaskParams{} + if err := taskParams.FromMap(params); err != nil { return fmt.Errorf("failed to parse task params: %w", err) } // Create and execute test task - testTask := NewTestTask(*params) + testTask := NewTestTask(*taskParams) return testTask.Execute(ctx, taskID, db) } diff --git a/task/worker.go b/task/worker.go index 6a47b56..280b217 100644 --- a/task/worker.go +++ b/task/worker.go @@ -14,11 +14,14 @@ import ( "modelRT/logger" "modelRT/mq" "modelRT/orm" - "modelRT/util" "github.com/gofrs/uuid" "github.com/panjf2000/ants/v2" amqp "github.com/rabbitmq/amqp091-go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" + oteltrace "go.opentelemetry.io/otel/trace" "gorm.io/gorm" ) @@ -283,14 +286,15 @@ func (w *TaskWorker) handleMessage(msg amqp.Delivery) { return } - // derive a per-task context carrying the trace propagated from the originating HTTP request - traceID := taskMsg.TraceID - if traceID == "" { - traceID = taskMsg.TaskID.String() // fallback when message carries no trace - } - taskCtx := context.WithValue(ctx, constants.CtxKeyTraceID, traceID) - taskCtx = context.WithValue(taskCtx, constants.CtxKeySpanID, util.GenerateSpanID("task-worker")) - taskCtx = context.WithValue(taskCtx, constants.CtxKeyParentSpanID, taskMsg.SpanID) + // Restore trace context from the publish span, then create an execute child span + taskCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(taskMsg.TraceCarrier)) + taskCtx, span := otel.Tracer("modelRT/task").Start(taskCtx, "task.execute", + oteltrace.WithAttributes( + attribute.String("task_id", taskMsg.TaskID.String()), + attribute.String("task_type", string(taskMsg.TaskType)), + ), + ) + defer span.End() ctx = taskCtx logger.Info(ctx, "Processing task", @@ -312,7 +316,7 @@ func (w *TaskWorker) handleMessage(msg amqp.Delivery) { // Execute task using handler startTime := time.Now() - err := w.handler.Execute(ctx, taskMsg.TaskID, taskMsg.TaskType, w.db) + err := w.handler.Execute(ctx, taskMsg.TaskID, taskMsg.TaskType, taskMsg.Params, w.db) processingTime := time.Since(startTime) if err != nil { From 33f7d758e55eea18ca28cc68f24ea2aa5e4e209a Mon Sep 17 00:00:00 2001 From: douxu Date: Mon, 27 Apr 2026 17:55:38 +0800 Subject: [PATCH 32/43] refactor: overhaul async task handler routing and fix data consistency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fix params lost in RabbitMQ transit by threading them through PublishTask/PublishTaskWithRetry - fix UpdateTaskErrorInfo not setting status=FAILED on async_task - fix UpdateAsyncTaskResultWithError silently skipping when no result row exists (UPDATE → upsert) - sync task failure to async_task_result in updateTaskWithError - remove taskType from AsyncTaskHandler.Execute interface; rename TaskHandler → AsyncTaskHandler - replace CompositeHandler with direct factory.GetHandler dispatch via worker.dispatch() - use constructors (NewXxxHandler) for handler registration instead of zero-value literals - consolidate TaskType/TaskStatus/UnifiedTaskType into task/types.go; delete types_v2.go - extract BaseTask/TaskParams into task/base_task.go --- database/async_task_extended.go | 1 + database/async_task_operations.go | 16 +-- task/base_task.go | 61 +++++++++++ task/handler_factory.go | 80 +++++--------- task/initializer.go | 6 +- task/queue_message.go | 20 +--- task/queue_producer.go | 9 +- task/retry_queue.go | 2 +- task/test_task.go | 19 ++-- task/types.go | 60 ++++------- task/types_v2.go | 138 ------------------------ task/worker.go | 171 ++++++++++++++++-------------- 12 files changed, 226 insertions(+), 357 deletions(-) create mode 100644 task/base_task.go delete mode 100644 task/types_v2.go diff --git a/database/async_task_extended.go b/database/async_task_extended.go index ca94b42..8d5849c 100644 --- a/database/async_task_extended.go +++ b/database/async_task_extended.go @@ -60,6 +60,7 @@ func UpdateTaskErrorInfo(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, err Updates(map[string]any{ "failure_reason": errorMsg, "stack_trace": stackTrace, + "status": orm.AsyncTaskStatusFailed, }) return result.Error diff --git a/database/async_task_operations.go b/database/async_task_operations.go index 991e77b..fce150b 100644 --- a/database/async_task_operations.go +++ b/database/async_task_operations.go @@ -161,14 +161,18 @@ func CreateAsyncTaskResult(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, r return resultOp.Error } -// UpdateAsyncTaskResultWithError updates a task result with error information +// UpdateAsyncTaskResultWithError upserts a task result with error information. func UpdateAsyncTaskResultWithError(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, code int, message string, detail orm.JSONMap) error { - // ctx timeout judgment cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - // Update with error information - result := tx.WithContext(cancelCtx). + if err := tx.WithContext(cancelCtx). + Where("task_id = ?", taskID). + FirstOrCreate(&orm.AsyncTaskResult{TaskID: taskID}).Error; err != nil { + return err + } + + return tx.WithContext(cancelCtx). Model(&orm.AsyncTaskResult{}). Where("task_id = ?", taskID). Updates(map[string]any{ @@ -176,9 +180,7 @@ func UpdateAsyncTaskResultWithError(ctx context.Context, tx *gorm.DB, taskID uui "error_message": message, "error_detail": detail, "result": nil, - }) - - return result.Error + }).Error } // UpdateAsyncTaskResultWithSuccess updates a task result with success information diff --git a/task/base_task.go b/task/base_task.go new file mode 100644 index 0000000..271d043 --- /dev/null +++ b/task/base_task.go @@ -0,0 +1,61 @@ +// Package task provides unified task type definitions and interfaces +package task + +import ( + "context" + "fmt" + + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// TaskParams defines the interface for task-specific parameters +type TaskParams interface { + Validate() error + GetType() UnifiedTaskType + ToMap() map[string]interface{} + FromMap(params map[string]interface{}) error +} + +// BaseTask provides common functionality for all task implementations +type BaseTask struct { + taskType UnifiedTaskType + params TaskParams + name string +} + +// NewBaseTask creates a new BaseTask instance +func NewBaseTask(taskType UnifiedTaskType, params TaskParams, name string) *BaseTask { + return &BaseTask{ + taskType: taskType, + params: params, + name: name, + } +} + +func (t *BaseTask) GetType() UnifiedTaskType { + return t.taskType +} + +func (t *BaseTask) GetParams() TaskParams { + return t.params +} + +func (t *BaseTask) GetName() string { + return t.name +} + +func (t *BaseTask) Validate() error { + if t.params == nil { + return fmt.Errorf("task parameters cannot be nil") + } + if t.taskType != t.params.GetType() { + return fmt.Errorf("task type mismatch: expected %s, got %s", t.taskType, t.params.GetType()) + } + return t.params.Validate() +} + +// Execute is a placeholder; concrete task types override this via embedding. +func (t *BaseTask) Execute(_ context.Context, _ uuid.UUID, _ *gorm.DB) error { + return fmt.Errorf("Execute not implemented for task type %s", t.taskType) +} diff --git a/task/handler_factory.go b/task/handler_factory.go index ae1d951..7fcc174 100644 --- a/task/handler_factory.go +++ b/task/handler_factory.go @@ -15,10 +15,10 @@ import ( "gorm.io/gorm" ) -// TaskHandler defines the interface for task processors -type TaskHandler interface { - // Execute processes a task with the given ID, type, and params from the MQ message - Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, params map[string]any, db *gorm.DB) error +// AsyncTaskHandler defines the interface for task processors +type AsyncTaskHandler interface { + // Execute processes a task with the given ID and params from the MQ message + Execute(ctx context.Context, taskID uuid.UUID, params map[string]any, db *gorm.DB) error // CanHandle returns true if this handler can process the given task type CanHandle(taskType TaskType) bool // Name returns the name of the handler for logging and metrics @@ -27,19 +27,19 @@ type TaskHandler interface { // HandlerFactory creates task handlers based on task type type HandlerFactory struct { - handlers map[TaskType]TaskHandler + handlers map[TaskType]AsyncTaskHandler mu sync.RWMutex } // NewHandlerFactory creates a new HandlerFactory func NewHandlerFactory() *HandlerFactory { return &HandlerFactory{ - handlers: make(map[TaskType]TaskHandler), + handlers: make(map[TaskType]AsyncTaskHandler), } } // RegisterHandler registers a handler for a specific task type -func (f *HandlerFactory) RegisterHandler(ctx context.Context, taskType TaskType, handler TaskHandler) { +func (f *HandlerFactory) RegisterHandler(ctx context.Context, taskType TaskType, handler AsyncTaskHandler) { f.mu.Lock() defer f.mu.Unlock() @@ -51,7 +51,7 @@ func (f *HandlerFactory) RegisterHandler(ctx context.Context, taskType TaskType, } // GetHandler returns a handler for the given task type -func (f *HandlerFactory) GetHandler(taskType TaskType) (TaskHandler, error) { +func (f *HandlerFactory) GetHandler(taskType TaskType) (AsyncTaskHandler, error) { f.mu.RLock() handler, exists := f.handlers[taskType] f.mu.RUnlock() @@ -65,10 +65,10 @@ func (f *HandlerFactory) GetHandler(taskType TaskType) (TaskHandler, error) { // CreateDefaultHandlers registers all default task handlers func (f *HandlerFactory) CreateDefaultHandlers(ctx context.Context) { - f.RegisterHandler(ctx, TypeTopologyAnalysis, &TopologyAnalysisHandler{}) - f.RegisterHandler(ctx, TypeEventAnalysis, &EventAnalysisHandler{}) - f.RegisterHandler(ctx, TypeBatchImport, &BatchImportHandler{}) - f.RegisterHandler(ctx, TaskType(TaskTypeTest), NewTestTaskHandler()) + f.RegisterHandler(ctx, TypeTopologyAnalysis, NewTopologyAnalysisHandler()) + f.RegisterHandler(ctx, TypeEventAnalysis, NewEventAnalysisHandler()) + f.RegisterHandler(ctx, TypeBatchImport, NewBatchImportHandler()) + f.RegisterHandler(ctx, TypeTest, NewTestTaskHandler()) } // BaseHandler provides common functionality for all task handlers @@ -103,7 +103,7 @@ func NewTopologyAnalysisHandler() *TopologyAnalysisHandler { // - start_component_uuid (string, required): BFS origin // - end_component_uuid (string, required): reachability target // - check_in_service (bool, optional, default true): skip out-of-service components -func (h *TopologyAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, params map[string]any, db *gorm.DB) error { +func (h *TopologyAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, params map[string]any, db *gorm.DB) error { logger.Info(ctx, "topology analysis started", "task_id", taskID) // Phase 1: parse params from MQ message @@ -158,6 +158,7 @@ func (h *TopologyAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, // check the start node itself before BFS if !inServiceMap[startComponentUUID] { + fmt.Println(11111) return persistTopologyResult(ctx, db, taskID, startComponentUUID, endComponentUUID, checkInService, false, nil, &startComponentUUID) } @@ -220,6 +221,7 @@ func (h *TopologyAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, // parseTopologyAnalysisParams extracts and validates the three required fields. // check_in_service defaults to true when absent. func parseTopologyAnalysisParams(params map[string]any) (startID, endID uuid.UUID, checkInService bool, err error) { + fmt.Printf("params:%+v\n", params) startStr, ok := params["start_component_uuid"].(string) if !ok || startStr == "" { err = fmt.Errorf("missing or invalid start_component_uuid") @@ -318,10 +320,10 @@ func NewEventAnalysisHandler() *EventAnalysisHandler { } // Execute processes an event analysis task -func (h *EventAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, params map[string]any, db *gorm.DB) error { +func (h *EventAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, params map[string]any, db *gorm.DB) error { logger.Info(ctx, "Starting event analysis", "task_id", taskID, - "task_type", taskType, + "task_params", params, ) // TODO: Implement actual event analysis logic @@ -334,7 +336,8 @@ func (h *EventAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, ta // Simulate work logger.Info(ctx, "Event analysis completed", "task_id", taskID, - "task_type", taskType, + "task_params", params, + "db", db, ) return nil @@ -358,10 +361,11 @@ func NewBatchImportHandler() *BatchImportHandler { } // Execute processes a batch import task -func (h *BatchImportHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, params map[string]any, db *gorm.DB) error { +func (h *BatchImportHandler) Execute(ctx context.Context, taskID uuid.UUID, params map[string]any, db *gorm.DB) error { logger.Info(ctx, "Starting batch import", "task_id", taskID, - "task_type", taskType, + "task_params", params, + "db", db, ) // TODO: Implement actual batch import logic @@ -374,7 +378,8 @@ func (h *BatchImportHandler) Execute(ctx context.Context, taskID uuid.UUID, task // Simulate work logger.Info(ctx, "Batch import completed", "task_id", taskID, - "task_type", taskType, + "task_params", params, + "db", db, ) return nil @@ -385,46 +390,9 @@ func (h *BatchImportHandler) CanHandle(taskType TaskType) bool { return taskType == TypeBatchImport } -// CompositeHandler can handle multiple task types by delegating to appropriate handlers -type CompositeHandler struct { - factory *HandlerFactory -} - -// NewCompositeHandler creates a new CompositeHandler -func NewCompositeHandler(factory *HandlerFactory) *CompositeHandler { - return &CompositeHandler{factory: factory} -} - -// Execute delegates task execution to the appropriate handler -func (h *CompositeHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, params map[string]any, db *gorm.DB) error { - handler, err := h.factory.GetHandler(taskType) - if err != nil { - return fmt.Errorf("failed to get handler for task type %s: %w", taskType, err) - } - - return handler.Execute(ctx, taskID, taskType, params, db) -} - -// CanHandle returns true if any registered handler can handle the task type -func (h *CompositeHandler) CanHandle(taskType TaskType) bool { - _, err := h.factory.GetHandler(taskType) - return err == nil -} - -// Name returns the composite handler name -func (h *CompositeHandler) Name() string { - return "composite_handler" -} - // DefaultHandlerFactory returns a HandlerFactory with all default handlers registered func DefaultHandlerFactory(ctx context.Context) *HandlerFactory { factory := NewHandlerFactory() factory.CreateDefaultHandlers(ctx) return factory } - -// DefaultCompositeHandler returns a CompositeHandler with all default handlers -func DefaultCompositeHandler(ctx context.Context) TaskHandler { - factory := DefaultHandlerFactory(ctx) - return NewCompositeHandler(factory) -} \ No newline at end of file diff --git a/task/initializer.go b/task/initializer.go index 1122a05..008d721 100644 --- a/task/initializer.go +++ b/task/initializer.go @@ -22,12 +22,10 @@ func InitTaskWorker(ctx context.Context, config config.ModelRTConfig, db *gorm.D } // Create task handler factory - handlerFactory := NewHandlerFactory() - handlerFactory.CreateDefaultHandlers(ctx) - handler := DefaultCompositeHandler(ctx) + handlerFactory := DefaultHandlerFactory(ctx) // Create task worker - worker, err := NewTaskWorker(ctx, workerCfg, db, config.RabbitMQConfig, handler) + worker, err := NewTaskWorker(ctx, workerCfg, db, config.RabbitMQConfig, handlerFactory) if err != nil { return nil, fmt.Errorf("failed to create task worker: %w", err) } diff --git a/task/queue_message.go b/task/queue_message.go index 9cc5ebf..c87c02c 100644 --- a/task/queue_message.go +++ b/task/queue_message.go @@ -9,16 +9,14 @@ import ( ) // TaskQueueMessage defines minimal message structure for RabbitMQ/Redis queue dispatch -// This struct is designed to be lightweight for efficient message transport type TaskQueueMessage struct { - TaskID uuid.UUID `json:"task_id"` - TaskType TaskType `json:"task_type"` - Priority int `json:"priority,omitempty"` // Optional, defaults to constants.TaskPriorityDefault - TraceCarrier map[string]string `json:"trace_carrier,omitempty"` // OTel propagation carrier (B3 headers) - Params map[string]any `json:"params,omitempty"` // Task-specific parameters, set by the HTTP handler + TaskID uuid.UUID `json:"task_id"` + TaskType TaskType `json:"task_type"` + Priority int `json:"priority,omitempty"` + TraceCarrier map[string]string `json:"trace_carrier,omitempty"` + Params map[string]any `json:"params,omitempty"` } -// NewTaskQueueMessage creates a new TaskQueueMessage with default priority func NewTaskQueueMessage(taskID uuid.UUID, taskType TaskType) *TaskQueueMessage { return &TaskQueueMessage{ TaskID: taskID, @@ -27,7 +25,6 @@ func NewTaskQueueMessage(taskID uuid.UUID, taskType TaskType) *TaskQueueMessage } } -// NewTaskQueueMessageWithPriority creates a new TaskQueueMessage with specified priority func NewTaskQueueMessageWithPriority(taskID uuid.UUID, taskType TaskType, priority int) *TaskQueueMessage { return &TaskQueueMessage{ TaskID: taskID, @@ -36,19 +33,14 @@ func NewTaskQueueMessageWithPriority(taskID uuid.UUID, taskType TaskType, priori } } -// ToJSON converts TaskQueueMessage to JSON bytes func (m *TaskQueueMessage) ToJSON() ([]byte, error) { return json.Marshal(m) } -// Validate checks if TaskQueueMessage is valid func (m *TaskQueueMessage) Validate() bool { - // Check if TaskID is valid (not nil UUID) if m.TaskID == uuid.Nil { return false } - - // Check if TaskType is valid switch m.TaskType { case TypeTopologyAnalysis, TypeEventAnalysis, TypeBatchImport, TypeTest: return true @@ -57,7 +49,6 @@ func (m *TaskQueueMessage) Validate() bool { } } -// SetPriority sets priority of task queue message with validation func (m *TaskQueueMessage) SetPriority(priority int) { if priority < constants.TaskPriorityLow { priority = constants.TaskPriorityLow @@ -68,7 +59,6 @@ func (m *TaskQueueMessage) SetPriority(priority int) { m.Priority = priority } -// GetPriority returns priority of task queue message func (m *TaskQueueMessage) GetPriority() int { return m.Priority } diff --git a/task/queue_producer.go b/task/queue_producer.go index bcaaffc..3ae6185 100644 --- a/task/queue_producer.go +++ b/task/queue_producer.go @@ -110,8 +110,9 @@ func (p *QueueProducer) declareInfrastructure() error { } // PublishTask publishes a task message to RabbitMQ -func (p *QueueProducer) PublishTask(ctx context.Context, taskID uuid.UUID, taskType TaskType, priority int) error { +func (p *QueueProducer) PublishTask(ctx context.Context, taskID uuid.UUID, taskType TaskType, priority int, params map[string]any) error { message := NewTaskQueueMessageWithPriority(taskID, taskType, priority) + message.Params = params // Validate message if !message.Validate() { @@ -166,10 +167,10 @@ func (p *QueueProducer) PublishTask(ctx context.Context, taskID uuid.UUID, taskT } // PublishTaskWithRetry publishes a task with retry logic -func (p *QueueProducer) PublishTaskWithRetry(ctx context.Context, taskID uuid.UUID, taskType TaskType, priority int, maxRetries int) error { +func (p *QueueProducer) PublishTaskWithRetry(ctx context.Context, taskID uuid.UUID, taskType TaskType, priority int, params map[string]any, maxRetries int) error { var lastErr error for i := range maxRetries { - err := p.PublishTask(ctx, taskID, taskType, priority) + err := p.PublishTask(ctx, taskID, taskType, priority, params) if err == nil { return nil } @@ -255,7 +256,7 @@ func PushTaskToRabbitMQ(ctx context.Context, cfg config.RabbitMQConfig, taskChan taskCtx, pubSpan := otel.Tracer("modelRT/task").Start(taskCtx, "task.publish", oteltrace.WithAttributes(attribute.String("task_id", msg.TaskID.String())), ) - if err := producer.PublishTaskWithRetry(taskCtx, msg.TaskID, msg.TaskType, msg.Priority, 3); err != nil { + if err := producer.PublishTaskWithRetry(taskCtx, msg.TaskID, msg.TaskType, msg.Priority, msg.Params, 3); err != nil { pubSpan.RecordError(err) logger.Error(taskCtx, "publish task to RabbitMQ failed", "task_id", msg.TaskID, "error", err) diff --git a/task/retry_queue.go b/task/retry_queue.go index c602f66..34499bd 100644 --- a/task/retry_queue.go +++ b/task/retry_queue.go @@ -120,7 +120,7 @@ func (q *RetryQueue) ProcessRetryQueue(ctx context.Context, batchSize int) error default: // Publish task to queue for immediate processing taskType := TaskType(task.TaskType) - if err := q.producer.PublishTask(ctx, task.TaskID, taskType, task.Priority); err != nil { + if err := q.producer.PublishTask(ctx, task.TaskID, taskType, task.Priority, map[string]any(task.Params)); err != nil { logger.Error(ctx, "Failed to publish retry task to queue", "task_id", task.TaskID, "task_type", taskType, diff --git a/task/test_task.go b/task/test_task.go index d85b682..580d1a2 100644 --- a/task/test_task.go +++ b/task/test_task.go @@ -104,11 +104,11 @@ func (t *TestTask) Execute(ctx context.Context, taskID uuid.UUID, db *gorm.DB) e // Build result result := map[string]interface{}{ - "status": "completed", - "sleep_duration": params.SleepDuration, - "message": params.Message, - "executed_at": time.Now().Unix(), - "task_id": taskID.String(), + "status": "completed", + "sleep_duration": params.SleepDuration, + "message": params.Message, + "executed_at": time.Now().Unix(), + "task_id": taskID.String(), } // Save result to database @@ -130,21 +130,22 @@ func (t *TestTask) Execute(ctx context.Context, taskID uuid.UUID, db *gorm.DB) e // TestTaskHandler handles test task execution type TestTaskHandler struct { - *BaseHandler + BaseHandler } // NewTestTaskHandler creates a new TestTaskHandler func NewTestTaskHandler() *TestTaskHandler { return &TestTaskHandler{ - BaseHandler: NewBaseHandler("test_task_handler"), + BaseHandler: *NewBaseHandler("test_task_handler"), } } // Execute processes a test task using the unified task interface -func (h *TestTaskHandler) Execute(ctx context.Context, taskID uuid.UUID, taskType TaskType, params map[string]any, db *gorm.DB) error { +func (h *TestTaskHandler) Execute(ctx context.Context, taskID uuid.UUID, params map[string]any, db *gorm.DB) error { logger.Info(ctx, "Executing test task", "task_id", taskID, - "task_type", taskType, + "task_params", params, + "db", db, ) // Convert params from MQ message to TestTaskParams diff --git a/task/types.go b/task/types.go index be1f4f5..ef59f6c 100644 --- a/task/types.go +++ b/task/types.go @@ -1,19 +1,6 @@ package task -import ( - "time" -) - -type TaskStatus string - -const ( - StatusPending TaskStatus = "PENDING" - StatusRunning TaskStatus = "RUNNING" - StatusCompleted TaskStatus = "COMPLETED" - StatusFailed TaskStatus = "FAILED" -) - -// TaskType 定义异步任务的具体业务类型 +// TaskType defines the business type of an async task type TaskType string const ( @@ -23,34 +10,23 @@ const ( TypeTest TaskType = "TEST" ) -type Task struct { - ID string `bson:"_id" json:"id"` - Type TaskType `bson:"type" json:"type"` - Status TaskStatus `bson:"status" json:"status"` - Priority int `bson:"priority" json:"priority"` +// TaskStatus defines the lifecycle status of an async task +type TaskStatus string - Params map[string]interface{} `bson:"params" json:"params"` - Result map[string]interface{} `bson:"result,omitempty" json:"result"` - ErrorMsg string `bson:"error_msg,omitempty" json:"error_msg"` +const ( + StatusPending TaskStatus = "PENDING" + StatusRunning TaskStatus = "RUNNING" + StatusCompleted TaskStatus = "COMPLETED" + StatusFailed TaskStatus = "FAILED" +) - CreatedAt time.Time `bson:"created_at" json:"created_at"` - StartedAt time.Time `bson:"started_at,omitempty" json:"started_at"` - CompletedAt time.Time `bson:"completed_at,omitempty" json:"completed_at"` -} +// UnifiedTaskType defines all async task types in a single location +type UnifiedTaskType string -type TopologyParams struct { - CheckIsland bool `json:"check_island"` - CheckShort bool `json:"check_short"` - BaseModelIDs []string `json:"base_model_ids"` -} - -type EventAnalysisParams struct { - MotorID string `json:"motor_id"` - TriggerID string `json:"trigger_id"` - DurationMS int `json:"duration_ms"` -} - -type BatchImportParams struct { - FileName string `json:"file_name"` - FilePath string `json:"file_path"` -} +const ( + TaskTypeTopologyAnalysis UnifiedTaskType = "TOPOLOGY_ANALYSIS" + TaskTypePerformanceAnalysis UnifiedTaskType = "PERFORMANCE_ANALYSIS" + TaskTypeEventAnalysis UnifiedTaskType = "EVENT_ANALYSIS" + TaskTypeBatchImport UnifiedTaskType = "BATCH_IMPORT" + TaskTypeTest UnifiedTaskType = "TEST" +) diff --git a/task/types_v2.go b/task/types_v2.go deleted file mode 100644 index aed3558..0000000 --- a/task/types_v2.go +++ /dev/null @@ -1,138 +0,0 @@ -// Package task provides unified task type definitions and interfaces -package task - -import ( - "context" - "fmt" - - "github.com/gofrs/uuid" - "gorm.io/gorm" -) - -// UnifiedTaskType defines all async task types in a single location -type UnifiedTaskType string - -const ( - // TaskTypeTopologyAnalysis represents topology analysis task - TaskTypeTopologyAnalysis UnifiedTaskType = "TOPOLOGY_ANALYSIS" - // TaskTypePerformanceAnalysis represents performance analysis task - TaskTypePerformanceAnalysis UnifiedTaskType = "PERFORMANCE_ANALYSIS" - // TaskTypeEventAnalysis represents event analysis task - TaskTypeEventAnalysis UnifiedTaskType = "EVENT_ANALYSIS" - // TaskTypeBatchImport represents batch import task - TaskTypeBatchImport UnifiedTaskType = "BATCH_IMPORT" - // TaskTypeTest represents test task for system verification - TaskTypeTest UnifiedTaskType = "TEST" -) - -// UnifiedTaskStatus defines task status constants -type UnifiedTaskStatus string - -const ( - // TaskStatusPending represents task waiting to be processed - TaskStatusPending UnifiedTaskStatus = "PENDING" - // TaskStatusRunning represents task currently executing - TaskStatusRunning UnifiedTaskStatus = "RUNNING" - // TaskStatusCompleted represents task finished successfully - TaskStatusCompleted UnifiedTaskStatus = "COMPLETED" - // TaskStatusFailed represents task failed with error - TaskStatusFailed UnifiedTaskStatus = "FAILED" -) - -// TaskParams defines the interface for task-specific parameters -// All task types must implement this interface to provide their parameter structure -type TaskParams interface { - // Validate checks if the parameters are valid for this task type - Validate() error - // GetType returns the task type these parameters are for - GetType() UnifiedTaskType - // ToMap converts parameters to map for database storage - ToMap() map[string]interface{} - // FromMap populates parameters from map (for database retrieval) - FromMap(params map[string]interface{}) error -} - -// UnifiedTask defines the base interface that all tasks must implement -// This provides a clean abstraction for task execution and management -type UnifiedTask interface { - // GetType returns the task type - GetType() UnifiedTaskType - - // GetParams returns the task parameters - GetParams() TaskParams - - // Execute performs the actual task logic - Execute(ctx context.Context, taskID uuid.UUID, db *gorm.DB) error - - // GetName returns a human-readable task name for logging - GetName() string - - // Validate checks if the task is valid before execution - Validate() error -} - -// BaseTask provides common functionality for all task implementations -type BaseTask struct { - taskType UnifiedTaskType - params TaskParams - name string -} - -// NewBaseTask creates a new BaseTask instance -func NewBaseTask(taskType UnifiedTaskType, params TaskParams, name string) *BaseTask { - return &BaseTask{ - taskType: taskType, - params: params, - name: name, - } -} - -// GetType returns the task type -func (t *BaseTask) GetType() UnifiedTaskType { - return t.taskType -} - -// GetParams returns the task parameters -func (t *BaseTask) GetParams() TaskParams { - return t.params -} - -// GetName returns the task name -func (t *BaseTask) GetName() string { - return t.name -} - -// Validate checks if the task is valid -func (t *BaseTask) Validate() error { - if t.params == nil { - return fmt.Errorf("task parameters cannot be nil") - } - - if t.taskType != t.params.GetType() { - return fmt.Errorf("task type mismatch: expected %s, got %s", t.taskType, t.params.GetType()) - } - - return t.params.Validate() -} - -// IsTaskType checks if a task type string is valid -func IsTaskType(taskType string) bool { - switch UnifiedTaskType(taskType) { - case TaskTypeTopologyAnalysis, TaskTypePerformanceAnalysis, - TaskTypeEventAnalysis, TaskTypeBatchImport, TaskTypeTest: - return true - default: - return false - } -} - -// GetTaskTypes returns all registered task types -func GetTaskTypes() []UnifiedTaskType { - return []UnifiedTaskType{ - TaskTypeTopologyAnalysis, - TaskTypePerformanceAnalysis, - TaskTypeEventAnalysis, - TaskTypeBatchImport, - TaskTypeTest, - } -} diff --git a/task/worker.go b/task/worker.go index 280b217..20795cf 100644 --- a/task/worker.go +++ b/task/worker.go @@ -52,56 +52,56 @@ func DefaultWorkerConfig() WorkerConfig { // TaskWorker manages a pool of workers for processing asynchronous tasks type TaskWorker struct { - cfg WorkerConfig - db *gorm.DB - pool *ants.Pool - conn *amqp.Connection - ch *amqp.Channel - handler TaskHandler - retryQueue *RetryQueue - stopChan chan struct{} - wg sync.WaitGroup - ctx context.Context - cancel context.CancelFunc - metrics *WorkerMetrics + cfg WorkerConfig + db *gorm.DB + pool *ants.Pool + conn *amqp.Connection + ch *amqp.Channel + factory *HandlerFactory + retryQueue *RetryQueue + stopChan chan struct{} + wg sync.WaitGroup + ctx context.Context + cancel context.CancelFunc + metrics *WorkerMetrics } // WorkerMetrics holds metrics for the worker pool type WorkerMetrics struct { // Task statistics by type - TasksProcessed map[TaskType]int64 - TasksFailed map[TaskType]int64 - TasksSuccess map[TaskType]int64 - ProcessingTime map[TaskType]time.Duration + TasksProcessed map[TaskType]int64 + TasksFailed map[TaskType]int64 + TasksSuccess map[TaskType]int64 + ProcessingTime map[TaskType]time.Duration // Aggregate counters (maintained for backward compatibility) - TotalProcessed int64 - TotalFailed int64 - TotalSuccess int64 - TasksInProgress int32 + TotalProcessed int64 + TotalFailed int64 + TotalSuccess int64 + TasksInProgress int32 // Queue and latency metrics - QueueDepth int - QueueLatency time.Duration + QueueDepth int + QueueLatency time.Duration // Worker resource metrics - WorkersActive int - WorkersIdle int - MemoryUsage uint64 - CPULoad float64 + WorkersActive int + WorkersIdle int + MemoryUsage uint64 + CPULoad float64 // Time window metrics - LastMinuteRate float64 - Last5MinutesRate float64 - LastHourRate float64 + LastMinuteRate float64 + Last5MinutesRate float64 + LastHourRate float64 // Health and timing - LastHealthCheck time.Time - mu sync.RWMutex + LastHealthCheck time.Time + mu sync.RWMutex } // NewTaskWorker creates a new TaskWorker instance -func NewTaskWorker(ctx context.Context, cfg WorkerConfig, db *gorm.DB, rabbitCfg config.RabbitMQConfig, handler TaskHandler) (*TaskWorker, error) { +func NewTaskWorker(ctx context.Context, cfg WorkerConfig, db *gorm.DB, rabbitCfg config.RabbitMQConfig, factory *HandlerFactory) (*TaskWorker, error) { // Initialize RabbitMQ connection mq.InitRabbitProxy(ctx, rabbitCfg) conn := mq.GetConn() @@ -118,10 +118,10 @@ func NewTaskWorker(ctx context.Context, cfg WorkerConfig, db *gorm.DB, rabbitCfg // Declare queue (ensure it exists with proper arguments) _, err = ch.QueueDeclare( constants.TaskQueueName, // name - true, // durable - false, // delete when unused - false, // exclusive - false, // no-wait + true, // durable + false, // delete when unused + false, // exclusive + false, // no-wait amqp.Table{ "x-max-priority": constants.TaskMaxPriority, "x-message-ttl": constants.TaskDefaultMessageTTL.Milliseconds(), @@ -158,15 +158,15 @@ func NewTaskWorker(ctx context.Context, cfg WorkerConfig, db *gorm.DB, rabbitCfg pool: pool, conn: conn, ch: ch, - handler: handler, + factory: factory, stopChan: make(chan struct{}), ctx: ctxWithCancel, cancel: cancel, metrics: &WorkerMetrics{ - TasksProcessed: make(map[TaskType]int64), - TasksFailed: make(map[TaskType]int64), - TasksSuccess: make(map[TaskType]int64), - ProcessingTime: make(map[TaskType]time.Duration), + TasksProcessed: make(map[TaskType]int64), + TasksFailed: make(map[TaskType]int64), + TasksSuccess: make(map[TaskType]int64), + ProcessingTime: make(map[TaskType]time.Duration), LastHealthCheck: time.Now(), }, } @@ -203,13 +203,13 @@ func (w *TaskWorker) consumerLoop(consumerID int) { // Consume messages from the queue msgs, err := w.ch.Consume( - constants.TaskQueueName, // queue + constants.TaskQueueName, // queue fmt.Sprintf("worker-%d", consumerID), // consumer tag - false, // auto-ack - false, // exclusive - false, // no-local - false, // no-wait - nil, // args + false, // auto-ack + false, // exclusive + false, // no-local + false, // no-wait + nil, // args ) if err != nil { logger.Error(w.ctx, "Failed to start consumer", @@ -316,7 +316,7 @@ func (w *TaskWorker) handleMessage(msg amqp.Delivery) { // Execute task using handler startTime := time.Now() - err := w.handler.Execute(ctx, taskMsg.TaskID, taskMsg.TaskType, taskMsg.Params, w.db) + err := w.dispatch(ctx, taskMsg.TaskType, taskMsg.TaskID, taskMsg.Params, &msg) processingTime := time.Since(startTime) if err != nil { @@ -431,29 +431,37 @@ func (w *TaskWorker) updateTaskStatus(ctx context.Context, taskID uuid.UUID, sta return nil } -// updateTaskWithError updates a task with error information +// updateTaskWithError updates a task with error information in both async_task and async_task_result. func (w *TaskWorker) updateTaskWithError(ctx context.Context, taskID uuid.UUID, err error) error { - // Update task error information in database errorMsg := err.Error() stackTrace := fmt.Sprintf("%+v", err) - updateErr := database.UpdateTaskErrorInfo(ctx, w.db, taskID, errorMsg, stackTrace) - if updateErr != nil { - logger.Error(ctx, "Failed to update task error info", - "task_id", taskID, - "error", updateErr, - ) + if updateErr := database.UpdateTaskErrorInfo(ctx, w.db, taskID, errorMsg, stackTrace); updateErr != nil { + logger.Error(ctx, "Failed to update task error info", "task_id", taskID, "error", updateErr) return updateErr } - logger.Warn(ctx, "Task failed with error", - "task_id", taskID, - "error", errorMsg, - ) + if updateErr := database.UpdateAsyncTaskResultWithError(ctx, w.db, taskID, 500, errorMsg, nil); updateErr != nil { + logger.Error(ctx, "Failed to update task result with error", "task_id", taskID, "error", updateErr) + return updateErr + } + logger.Warn(ctx, "Task failed with error", "task_id", taskID, "error", errorMsg) return nil } +// dispatch routes a task message to the appropriate handler and executes it. +// Nacks the message and returns an error if no handler is registered for the task type. +func (w *TaskWorker) dispatch(ctx context.Context, taskType TaskType, taskID uuid.UUID, params map[string]any, msg *amqp.Delivery) error { + handler, err := w.factory.GetHandler(taskType) + if err != nil { + logger.Error(ctx, "No handler for task type", "task_type", taskType) + msg.Nack(false, false) + return err + } + return handler.Execute(ctx, taskID, params, w.db) +} + // healthCheckLoop periodically checks worker health and metrics func (w *TaskWorker) healthCheckLoop() { defer w.wg.Done() @@ -479,10 +487,10 @@ func (w *TaskWorker) checkHealth() { // Update queue depth queue, err := w.ch.QueueDeclarePassive( constants.TaskQueueName, // name - true, // durable - false, // delete when unused - false, // exclusive - false, // no-wait + true, // durable + false, // delete when unused + false, // exclusive + false, // no-wait amqp.Table{ "x-max-priority": constants.TaskMaxPriority, "x-message-ttl": constants.TaskDefaultMessageTTL.Milliseconds(), @@ -565,24 +573,24 @@ func (w *TaskWorker) GetMetrics() *WorkerMetrics { // Create a copy without the mutex to avoid copylocks warning return &WorkerMetrics{ - TasksProcessed: tasksProcessedCopy, - TasksFailed: tasksFailedCopy, - TasksSuccess: tasksSuccessCopy, - ProcessingTime: processingTimeCopy, - TotalProcessed: w.metrics.TotalProcessed, - TotalFailed: w.metrics.TotalFailed, - TotalSuccess: w.metrics.TotalSuccess, - TasksInProgress: w.metrics.TasksInProgress, - QueueDepth: w.metrics.QueueDepth, - QueueLatency: w.metrics.QueueLatency, - WorkersActive: w.metrics.WorkersActive, - WorkersIdle: w.metrics.WorkersIdle, - MemoryUsage: w.metrics.MemoryUsage, - CPULoad: w.metrics.CPULoad, - LastMinuteRate: w.metrics.LastMinuteRate, + TasksProcessed: tasksProcessedCopy, + TasksFailed: tasksFailedCopy, + TasksSuccess: tasksSuccessCopy, + ProcessingTime: processingTimeCopy, + TotalProcessed: w.metrics.TotalProcessed, + TotalFailed: w.metrics.TotalFailed, + TotalSuccess: w.metrics.TotalSuccess, + TasksInProgress: w.metrics.TasksInProgress, + QueueDepth: w.metrics.QueueDepth, + QueueLatency: w.metrics.QueueLatency, + WorkersActive: w.metrics.WorkersActive, + WorkersIdle: w.metrics.WorkersIdle, + MemoryUsage: w.metrics.MemoryUsage, + CPULoad: w.metrics.CPULoad, + LastMinuteRate: w.metrics.LastMinuteRate, Last5MinutesRate: w.metrics.Last5MinutesRate, - LastHourRate: w.metrics.LastHourRate, - LastHealthCheck: w.metrics.LastHealthCheck, + LastHourRate: w.metrics.LastHourRate, + LastHealthCheck: w.metrics.LastHealthCheck, // Mutex is intentionally omitted } } @@ -595,6 +603,7 @@ func (w *TaskWorker) IsHealthy() bool { // Consider unhealthy if last health check was too long ago return time.Since(w.metrics.LastHealthCheck) < 2*w.cfg.PollingInterval } + // RecordMetrics periodically records worker metrics to the logging system func (w *TaskWorker) RecordMetrics(interval time.Duration) { go func() { From 966127893526a3648233f8d2c64a121fd775a1f8 Mon Sep 17 00:00:00 2001 From: douxu Date: Tue, 28 Apr 2026 17:41:28 +0800 Subject: [PATCH 33/43] refactor: rename TaskParams to Params and remove debug prints - rename TaskParams interface to Params in task/base_task.go for brevity - remove debug fmt.Println/Printf statements from graph.go and handler_factory.go - fix is_local flag from false to true for existing test components in deploy.md - add 6 new test component records (ns4-ns8) to deploy seed data --- common/errcode/bussiness_error.go | 6 + deploy/deploy.md | 71 ++- diagram/graph.go | 1 - doc/async_task_api.md | 539 ++++++++++++++++++ handler/async_task_cancel_handler.go | 55 +- handler/async_task_progress_update_handler.go | 25 +- handler/async_task_result_detail_handler.go | 49 +- handler/async_task_result_query_handler.go | 53 +- handler/async_task_status_update_handler.go | 37 +- task/base_task.go | 10 +- task/handler_factory.go | 2 - 11 files changed, 661 insertions(+), 187 deletions(-) create mode 100644 doc/async_task_api.md diff --git a/common/errcode/bussiness_error.go b/common/errcode/bussiness_error.go index 08fcfea..6ab5999 100644 --- a/common/errcode/bussiness_error.go +++ b/common/errcode/bussiness_error.go @@ -46,4 +46,10 @@ var ( // ErrCacheQueryFailed define variable to indicates query cached data by token failed. ErrCacheQueryFailed = newError(60003, "query cached data by token failed") + + // ErrTaskNotFound indicates the async task with the given ID does not exist. + ErrTaskNotFound = newError(40008, "async task not found") + + // ErrTaskCannotCancel indicates the task is already running or completed and cannot be cancelled. + ErrTaskCannotCancel = newError(40009, "task cannot be cancelled, already running or completed") ) diff --git a/deploy/deploy.md b/deploy/deploy.md index 3364838..0080b23 100644 --- a/deploy/deploy.md +++ b/deploy/deploy.md @@ -136,7 +136,7 @@ VALUES 'ns1', 'tag1', 'component1', 'bus_1', '', 'grid1', 'zone1', 'station1', 1, -1, - false, + true, -1, -1, '{}', '{}', @@ -149,7 +149,7 @@ VALUES 'ns2', 'tag2', 'component2', 'bus_1', '', 'grid1', 'zone1', 'station1', 1, -1, - false, + true, -1, -1, '{}', '{}', @@ -162,13 +162,78 @@ VALUES 'ns3', 'tag3', 'component3', 'bus_1', '', 'grid1', 'zone1', 'station2', 2, -1, - false, + true, -1, -1, '{}', '{}', '{}', -1, CURRENT_TIMESTAMP +), +( + '70c190f2-8a60-42a9-b143-ec5f87e0aa6b', + 'ns4', 'tag4', 'component4', 'bus_1', '', + 'grid1', 'zone1', 'station1', 1, + -1, + true, + -1, -1, + '{}', + '{}', + '{}', + -1, + CURRENT_TIMESTAMP +), +( + '10f155cf-bd27-4557-85b2-d126b6e2657f', + 'ns5', 'tag5', 'component5', 'bus_1', '', + 'grid1', 'zone1', 'station1', 1, + -1, + true, + -1, -1, + '{}', + '{}', + '{}', + -1, + CURRENT_TIMESTAMP +), +( + 'e32bc0be-67f4-4d79-a5da-eaa40a5bd77d', + 'ns6', 'tag6', 'component6', 'bus_1', '', + 'grid1', 'zone1', 'station1', 1, + -1, + true, + -1, -1, + '{}', + '{}', + '{}', + -1, + CURRENT_TIMESTAMP +), +( + '70c190f2-8a75-42a9-b166-ec5f87e0aa6b', + 'ns7', 'tag7', 'component7', 'bus_1', '', + 'grid1', 'zone1', 'station1', 1, + -1, + true, + -1, -1, + '{}', + '{}', + '{}', + -1, + CURRENT_TIMESTAMP +), +( + '70c200f2-8a75-42a9-c166-bf5f87e0aa6b', + 'ns8', 'tag8', 'component8', 'bus_1', '', + 'grid1', 'zone1', 'station1', 1, + -1, + true, + -1, -1, + '{}', + '{}', + '{}', + -1, + CURRENT_TIMESTAMP ); INSERT INTO public.measurement (id, tag, name, type, size, data_source, event_plan, bay_uuid, component_uuid, op, ts) diff --git a/diagram/graph.go b/diagram/graph.go index 1d9ee6e..6c4970d 100644 --- a/diagram/graph.go +++ b/diagram/graph.go @@ -112,7 +112,6 @@ func (g *Graph) DelEdge(from, to uuid.UUID) error { return fmt.Errorf("delete edge failed: %w", err) } - fmt.Println("fromKeys:", fromKeys) for _, fromUUID := range fromKeys { fromKey := fromUUID.String() var delIndex int diff --git a/doc/async_task_api.md b/doc/async_task_api.md new file mode 100644 index 0000000..6a474ed --- /dev/null +++ b/doc/async_task_api.md @@ -0,0 +1,539 @@ +# ModelRT 异步任务 API 文档 + +## 1. 概述 + +ModelRT 异步任务系统基于 RabbitMQ 消息驱动，提供完整的任务生命周期管理。任务创建后进入队列，由后台 Worker 消费并执行，调用方可通过轮询接口获取进度和结果。 + +**Base URL**: `http://{host}:{port}`（默认 `localhost:8080`） + +**鉴权**: 公开接口需在 Header 中携带 `X-Service-Token`（由服务端启动时生成）。 + +--- + +## 2. API 端点总览 + +| 方法 | 路径 | 描述 | 权限 | +| :----- | :--------------------------------- | :--------------- | :----- | +| POST | `/task/async` | 创建异步任务 | 公开 | +| GET | `/task/async/results` | 批量查询任务结果 | 公开 | +| GET | `/task/async/{task_id}` | 查询单个任务详情 | 公开 | +| POST | `/task/async/{task_id}/cancel` | 取消异步任务 | 公开 | +| POST | `/task/internal/async/progress` | 更新任务进度 | 内部 | +| POST | `/task/internal/async/status` | 更新任务状态 | 内部 | + +--- + +## 3. 通用响应结构 + +### 成功响应 + +```json +{ + "code": 2000, + "msg": "success message", + "payload": {} +} +``` + +### 失败响应 + +```json +{ + "code": 4001, + "msg": "error description", + "payload": null +} +``` + +### 响应码说明 + +| code | 含义 | +| :--- | :--------------------- | +| 2000 | 成功 | +| 3000 | 处理失败 | +| 4001 | 请求参数无效 | +| 4002 | 未授权（Token 无效） | +| 5000 | 服务器内部错误 | + +--- + +## 4. 详细接口说明 + +### 4.1 创建异步任务 + +**POST** `/task/async` + +创建新的异步任务，任务进入队列等待 Worker 消费。返回 `task_id` 用于后续查询。 + +**请求头** + +``` +Content-Type: application/json +X-Service-Token: +``` + +**请求体** + +```json +{ + "task_type": "TOPOLOGY_ANALYSIS", + "params": { } +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :---------- | :----- | :--- | :-------------------------------------------------------------------------------- | +| `task_type` | string | 是 | 任务类型，枚举值见 §5.1 | +| `params` | object | 是 | 任务参数，不同任务类型的参数结构见 §5.3 | + +**成功响应** `200` + +```json +{ + "code": 2000, + "msg": "task created successfully", + "payload": { + "task_id": "123e4567-e89b-12d3-a456-426614174000" + } +} +``` + +**失败响应** + +| 场景 | code | msg | +| :----------------- | :--- | :-------------------------- | +| 参数格式错误 | 4001 | invalid request parameters | +| task_type 不合法 | 4001 | invalid task type | +| params 内容不合法 | 4001 | invalid task parameters | +| 数据库连接失败 | 5000 | database connection error | +| 任务写库失败 | 5000 | failed to create task | + +**curl 示例** + +```bash +curl -X POST "http://localhost:8080/task/async" \ + -H "Content-Type: application/json" \ + -H "X-Service-Token: " \ + -d '{ + "task_type": "TOPOLOGY_ANALYSIS", + "params": { + "start_component_uuid": "550e8400-e29b-41d4-a716-446655440000", + "end_component_uuid": "550e8400-e29b-41d4-a716-446655440001", + "check_in_service": true + } + }' +``` + +--- + +### 4.2 批量查询任务结果 + +**GET** `/task/async/results` + +根据一组任务 ID 批量查询状态和结果，适用于轮询多个任务。 + +**Query 参数** + +| 参数 | 类型 | 必填 | 说明 | +| :--------- | :----- | :--- | :----------------------------------- | +| `task_ids` | string | 是 | 逗号分隔的 UUID 列表，最少 1 个 | + +**请求示例** + +``` +GET /task/async/results?task_ids=123e4567-e89b-12d3-a456-426614174000,223e4567-e89b-12d3-a456-426614174001 +``` + +**成功响应** `200` + +```json +{ + "code": 2000, + "msg": "query completed", + "payload": { + "total": 2, + "tasks": [ + { + "task_id": "123e4567-e89b-12d3-a456-426614174000", + "task_type": "TOPOLOGY_ANALYSIS", + "status": "RUNNING", + "progress": 50, + "created_at": 1741846200 + }, + { + "task_id": "223e4567-e89b-12d3-a456-426614174001", + "task_type": "PERFORMANCE_ANALYSIS", + "status": "COMPLETED", + "progress": 100, + "created_at": 1741846200, + "finished_at": 1741846260, + "result": { + "components_analyzed": 5 + } + } + ] + } +} +``` + +**失败响应** + +| 场景 | code | msg | +| :----------------- | :--- | :-------------------------- | +| 缺少 task_ids | 4001 | task_ids parameter is required | +| UUID 格式不合法 | 4001 | invalid task ID format | +| 数据库连接失败 | 5000 | database connection error | +| 查询失败 | 5000 | failed to query tasks | + +**curl 示例** + +```bash +curl "http://localhost:8080/task/async/results?task_ids=123e4567-e89b-12d3-a456-426614174000" +``` + +--- + +### 4.3 查询单个任务详情 + +**GET** `/task/async/{task_id}` + +查询单个任务的完整信息，包含结果或错误详情。 + +**路径参数** + +| 参数 | 类型 | 必填 | 说明 | +| :-------- | :----- | :--- | :-------------- | +| `task_id` | string | 是 | 任务 UUID | + +**成功响应** `200` + +```json +{ + "code": 2000, + "msg": "query completed", + "payload": { + "task_id": "123e4567-e89b-12d3-a456-426614174000", + "task_type": "TOPOLOGY_ANALYSIS", + "status": "COMPLETED", + "progress": 100, + "created_at": 1741846200, + "finished_at": 1741846260, + "result": { + "path_exists": true, + "path_length": 3, + "path_nodes": ["comp-001", "comp-005", "comp-999"] + } + } +} +``` + +任务失败时 payload 附带错误信息： + +```json +{ + "code": 2000, + "msg": "query completed", + "payload": { + "task_id": "123e4567-e89b-12d3-a456-426614174000", + "task_type": "TOPOLOGY_ANALYSIS", + "status": "FAILED", + "created_at": 1741846200, + "finished_at": 1741846210, + "error_code": 400102, + "error_message": "Component UUID not found", + "error_detail": { + "component_uuid": "550e8400-0000-0000-0000-000000000000" + } + } +} +``` + +**失败响应** + +| 场景 | code | msg | +| :----------------- | :--- | :-------------------------- | +| 缺少 task_id | 4001 | task_id parameter is required | +| UUID 格式不合法 | 4001 | invalid task ID format | +| 任务不存在 | 404 | task not found | +| 数据库连接失败 | 5000 | database connection error | + +**curl 示例** + +```bash +curl "http://localhost:8080/task/async/123e4567-e89b-12d3-a456-426614174000" +``` + +--- + +### 4.4 取消异步任务 + +**POST** `/task/async/{task_id}/cancel` + +取消指定任务。**仅 `SUBMITTED`（排队中）状态的任务可以被取消**，已开始执行（`RUNNING`）或已结束的任务无法取消。 + +取消后任务状态变为 `FAILED`，错误码 `40003`，错误信息 `task cancelled by user`。 + +**路径参数** + +| 参数 | 类型 | 必填 | 说明 | +| :-------- | :----- | :--- | :-------- | +| `task_id` | string | 是 | 任务 UUID | + +**成功响应** `200` + +```json +{ + "code": 2000, + "msg": "task cancelled successfully" +} +``` + +**失败响应** + +| 场景 | code | msg | +| :----------------------- | :--- | :--------------------------------------------------------- | +| 缺少 task_id | 400 | task_id parameter is required | +| UUID 格式不合法 | 400 | invalid task ID format | +| 任务不存在 | 404 | task not found | +| 任务已执行或已完成 | 400 | task cannot be cancelled (already running or completed) | +| 数据库连接失败 | 500 | database connection error | +| 取消操作失败 | 500 | failed to cancel task | + +**curl 示例** + +```bash +curl -X POST "http://localhost:8080/task/async/123e4567-e89b-12d3-a456-426614174000/cancel" +``` + +--- + +### 4.5 内部接口：更新任务进度 + +**POST** `/task/internal/async/progress` + +由 Worker 内部调用，更新任务执行进度（0-100）。 + +**请求体** + +```json +{ + "task_id": "123e4567-e89b-12d3-a456-426614174000", + "progress": 75 +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :--------- | :----- | :--- | :---------------- | +| `task_id` | string | 是 | 任务 UUID | +| `progress` | int | 是 | 进度值，范围 0-100 | + +**成功响应** `200` + +```json +{ + "code": 2000, + "msg": "task progress updated successfully", + "payload": null +} +``` + +--- + +### 4.6 内部接口：更新任务状态 + +**POST** `/task/internal/async/status` + +由 Worker 内部调用，更新任务状态。当状态更新为 `COMPLETED` 或 `FAILED` 时，同步写入 `finished_at` 时间戳。 + +**请求体** + +```json +{ + "task_id": "123e4567-e89b-12d3-a456-426614174000", + "status": "RUNNING", + "timestamp": 1741846205 +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :---------- | :----- | :--- | :--------------------------------------- | +| `task_id` | string | 是 | 任务 UUID | +| `status` | string | 是 | 目标状态，枚举值见 §5.2 | +| `timestamp` | int64 | 是 | 状态变更时间戳（Unix 秒） | + +**成功响应** `200` + +```json +{ + "code": 2000, + "msg": "task status updated successfully", + "payload": null +} +``` + +--- + +## 5. 数据结构参考 + +### 5.1 任务类型（task_type） + +| 枚举值 | 描述 | +| :--------------------- | :----------- | +| `TOPOLOGY_ANALYSIS` | 拓扑连通性分析 | +| `PERFORMANCE_ANALYSIS` | 性能分析 | +| `EVENT_ANALYSIS` | 事件分析 | +| `BATCH_IMPORT` | 批量数据导入 | +| `TEST` | 测试任务（系统验证用） | + +### 5.2 任务状态（status） + +| 枚举值 | 描述 | 可转换至 | +| :---------- | :----------------- | :------------------------------ | +| `SUBMITTED` | 已提交至队列 | `RUNNING`, `FAILED`（取消） | +| `RUNNING` | 正在执行 | `COMPLETED`, `FAILED` | +| `COMPLETED` | 执行成功 | — | +| `FAILED` | 执行失败或被取消 | — | + +### 5.3 各任务类型的 params 结构 + +#### TOPOLOGY_ANALYSIS — 拓扑连通性分析 + +分析两个元件之间是否存在连通路径。 + +```json +{ + "start_component_uuid": "550e8400-e29b-41d4-a716-446655440000", + "end_component_uuid": "550e8400-e29b-41d4-a716-446655440001", + "check_in_service": true +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :--------------------- | :------ | :--- | :------------------------------------- | +| `start_component_uuid` | string | 是 | 起始元件 UUID | +| `end_component_uuid` | string | 是 | 目标元件 UUID | +| `check_in_service` | boolean | 否 | 是否只检查投运状态元件，默认 `true` | + +#### PERFORMANCE_ANALYSIS — 性能分析 + +```json +{ + "component_ids": ["comp-001", "comp-002"], + "time_range": { + "start": "2026-03-01T00:00:00Z", + "end": "2026-03-02T00:00:00Z" + } +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :--------------- | :------- | :--- | :------------------ | +| `component_ids` | []string | 是 | 待分析的元件 ID 列表（至少 1 个） | +| `time_range` | object | 否 | 分析时间范围 | + +#### EVENT_ANALYSIS — 事件分析 + +```json +{ + "event_type": "MOTOR_START", + "start_time": "2026-03-01T00:00:00Z", + "end_time": "2026-03-02T00:00:00Z", + "components": ["comp-001", "comp-002"] +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :------------ | :------- | :--- | :------------------ | +| `event_type` | string | 是 | 事件类型 | +| `start_time` | string | 否 | 事件起始时间（RFC3339） | +| `end_time` | string | 否 | 事件截止时间（RFC3339） | +| `components` | []string | 否 | 关联的元件列表 | + +#### BATCH_IMPORT — 批量导入 + +```json +{ + "file_path": "/data/import/model.csv", + "file_type": "CSV", + "options": { + "overwrite": false, + "validate": true, + "notify_user": true + } +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :--------------------- | :------ | :--- | :-------------------------------- | +| `file_path` | string | 是 | 导入文件路径 | +| `file_type` | string | 否 | 文件类型：`CSV`, `JSON`, `XML` | +| `options.overwrite` | boolean | 否 | 是否覆盖已有数据，默认 `false` | +| `options.validate` | boolean | 否 | 是否校验数据，默认 `true` | +| `options.notify_user` | boolean | 否 | 完成后是否通知用户，默认 `true` | + +#### TEST — 测试任务 + +```json +{ + "sleep_duration": 30 +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :--------------- | :--- | :--- | :---------------------------------------------- | +| `sleep_duration` | int | 否 | 模拟执行耗时（秒），默认 60，最大 3600 | + +### 5.4 任务结果对象（AsyncTaskResult） + +| 字段 | 类型 | 说明 | +| :-------------- | :----- | :--------------------------------------------------- | +| `task_id` | string | 任务 UUID | +| `task_type` | string | 任务类型 | +| `status` | string | 当前状态 | +| `progress` | int | 进度（0-100），仅 `RUNNING` 时返回 | +| `created_at` | int64 | 创建时间戳（Unix 秒） | +| `finished_at` | int64 | 完成时间戳（Unix 秒），仅 `COMPLETED`/`FAILED` 返回 | +| `result` | object | 任务结果，仅 `COMPLETED` 时返回 | +| `error_code` | int | 错误码，仅 `FAILED` 时返回 | +| `error_message` | string | 错误描述，仅 `FAILED` 时返回 | +| `error_detail` | object | 错误详情，仅 `FAILED` 时返回 | + +--- + +## 6. 典型调用流程 + +``` +创建任务 + └─ POST /task/async + └─ 返回 task_id + +轮询状态（建议间隔 2-5 秒） + └─ GET /task/async/{task_id} + ├─ status=SUBMITTED → 继续等待 + ├─ status=RUNNING → 查看 progress + ├─ status=COMPLETED → 读取 result 字段 + └─ status=FAILED → 读取 error_code / error_message + +如需中止（仅 SUBMITTED 状态有效） + └─ POST /task/async/{task_id}/cancel +``` + +--- + +## 7. 队列配置参考 + +| 配置项 | 值 | +| :------------- | :-------------------------- | +| Exchange | `modelrt.tasks.exchange` | +| Queue | `modelrt.tasks.queue` | +| Routing Key | `modelrt.task` | +| 优先级范围 | 0–10（默认 5） | +| 消息 TTL | 24 小时 | +| 最大重试次数 | 3 次 | +| 重试初始延迟 | 1 秒（指数退避，最大 5 分钟）| + +--- + +**文档版本**: 1.1 +**最后更新**: 2026-04-28 +**相关文档**: [异步任务系统设计文档](./async_task_system.md) diff --git a/handler/async_task_cancel_handler.go b/handler/async_task_cancel_handler.go index 9b99b52..6fa0bc3 100644 --- a/handler/async_task_cancel_handler.go +++ b/handler/async_task_cancel_handler.go @@ -2,12 +2,11 @@ package handler import ( - "net/http" "time" + "modelRT/constants" "modelRT/database" "modelRT/logger" - "modelRT/network" "modelRT/orm" "github.com/gin-gonic/gin" @@ -23,97 +22,65 @@ import ( // @Produce json // @Param task_id path string true "任务ID" // @Success 200 {object} network.SuccessResponse "任务取消成功" -// @Failure 400 {object} network.FailureResponse "请求参数错误或任务无法取消" -// @Failure 404 {object} network.FailureResponse "任务不存在" -// @Failure 500 {object} network.FailureResponse "服务器内部错误" +// @Failure 200 {object} network.FailureResponse "请求参数错误或任务无法取消" // @Router /task/async/{task_id}/cancel [post] func AsyncTaskCancelHandler(c *gin.Context) { ctx := c.Request.Context() - // Parse task ID from path parameter taskIDStr := c.Param("task_id") if taskIDStr == "" { logger.Error(ctx, "task_id parameter is required") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "task_id parameter is required", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "task_id parameter is required", nil) return } taskID, err := uuid.FromString(taskIDStr) if err != nil { logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid task ID format", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid task ID format", nil) return } pgClient := database.GetPostgresDBClient() if pgClient == nil { logger.Error(ctx, "database connection not found in context") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "database connection error", - }) + renderRespFailure(c, constants.RespCodeServerError, "database connection error", nil) return } - // Query task from database asyncTask, err := database.GetAsyncTaskByID(ctx, pgClient, taskID) if err != nil { if err == gorm.ErrRecordNotFound { logger.Error(ctx, "async task not found", "task_id", taskID) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusNotFound, - Msg: "task not found", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "task not found", nil) return } logger.Error(ctx, "failed to query async task from database", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to query task", - }) + renderRespFailure(c, constants.RespCodeServerError, "failed to query task", nil) return } - // Check if task can be cancelled (only SUBMITTED tasks can be cancelled) if asyncTask.Status != orm.AsyncTaskStatusSubmitted { logger.Error(ctx, "task cannot be cancelled", "task_id", taskID, "status", asyncTask.Status) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "task cannot be cancelled (already running or completed)", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "task cannot be cancelled, already running or completed", nil) return } - // Update task status to failed with cancellation reason timestamp := time.Now().Unix() err = database.FailAsyncTask(ctx, pgClient, taskID, timestamp) if err != nil { logger.Error(ctx, "failed to cancel async task", "task_id", taskID, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to cancel task", - }) + renderRespFailure(c, constants.RespCodeServerError, "failed to cancel task", nil) return } - // Update task result with cancellation error - err = database.UpdateAsyncTaskResultWithError(ctx, pgClient, taskID, 40003, "task cancelled by user", orm.JSONMap{ + err = database.UpdateAsyncTaskResultWithError(ctx, pgClient, taskID, 40009, "task cancelled by user", orm.JSONMap{ "cancelled_at": timestamp, "cancelled_by": "user", }) if err != nil { logger.Error(ctx, "failed to update task result with cancellation error", "task_id", taskID, "error", err) - // Continue anyway since task is already marked as failed } - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: 2000, - Msg: "task cancelled successfully", - }) + renderRespSuccess(c, constants.RespCodeSuccess, "task cancelled successfully", nil) } diff --git a/handler/async_task_progress_update_handler.go b/handler/async_task_progress_update_handler.go index ad3fe9b..08276ce 100644 --- a/handler/async_task_progress_update_handler.go +++ b/handler/async_task_progress_update_handler.go @@ -2,8 +2,7 @@ package handler import ( - "net/http" - + "modelRT/constants" "modelRT/database" "modelRT/logger" "modelRT/network" @@ -18,37 +17,23 @@ func AsyncTaskProgressUpdateHandler(c *gin.Context) { if err := c.ShouldBindJSON(&request); err != nil { logger.Error(ctx, "failed to unmarshal async task progress update request", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid request parameters", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid request parameters", nil) return } pgClient := database.GetPostgresDBClient() if pgClient == nil { logger.Error(ctx, "database connection not found in context") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "database connection error", - }) + renderRespFailure(c, constants.RespCodeServerError, "database connection error", nil) return } - // Update task progress err := database.UpdateAsyncTaskProgress(ctx, pgClient, request.TaskID, request.Progress) if err != nil { logger.Error(ctx, "failed to update async task progress", "task_id", request.TaskID, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to update task progress", - }) + renderRespFailure(c, constants.RespCodeServerError, "failed to update task progress", nil) return } - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: 2000, - Msg: "task progress updated successfully", - Payload: nil, - }) + renderRespSuccess(c, constants.RespCodeSuccess, "task progress updated successfully", nil) } diff --git a/handler/async_task_result_detail_handler.go b/handler/async_task_result_detail_handler.go index 1494945..2d4174a 100644 --- a/handler/async_task_result_detail_handler.go +++ b/handler/async_task_result_detail_handler.go @@ -2,8 +2,7 @@ package handler import ( - "net/http" - + "modelRT/constants" "modelRT/database" "modelRT/logger" "modelRT/network" @@ -21,75 +20,51 @@ import ( // @Produce json // @Param task_id path string true "任务ID" // @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskResult} "查询成功" -// @Failure 400 {object} network.FailureResponse "请求参数错误" -// @Failure 404 {object} network.FailureResponse "任务不存在" -// @Failure 500 {object} network.FailureResponse "服务器内部错误" +// @Failure 200 {object} network.FailureResponse "请求参数错误" // @Router /task/async/{task_id} [get] func AsyncTaskResultDetailHandler(c *gin.Context) { ctx := c.Request.Context() - // Parse task ID from path parameter taskIDStr := c.Param("task_id") if taskIDStr == "" { logger.Error(ctx, "task_id parameter is required") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "task_id parameter is required", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "task_id parameter is required", nil) return } taskID, err := uuid.FromString(taskIDStr) if err != nil { logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid task ID format", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid task ID format", nil) return } pgClient := database.GetPostgresDBClient() if pgClient == nil { logger.Error(ctx, "database connection not found in context") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "database connection error", - }) + renderRespFailure(c, constants.RespCodeServerError, "database connection error", nil) return } - // Query task from database asyncTask, err := database.GetAsyncTaskByID(ctx, pgClient, taskID) if err != nil { if err == gorm.ErrRecordNotFound { logger.Error(ctx, "async task not found", "task_id", taskID) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusNotFound, - Msg: "task not found", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "task not found", nil) return } logger.Error(ctx, "failed to query async task from database", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to query task", - }) + renderRespFailure(c, constants.RespCodeServerError, "failed to query task", nil) return } - // Query task result from database taskResult, err := database.GetAsyncTaskResult(ctx, pgClient, taskID) if err != nil { logger.Error(ctx, "failed to query async task result from database", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to query task result", - }) + renderRespFailure(c, constants.RespCodeServerError, "failed to query task result", nil) return } - // Convert to response format responseTask := network.AsyncTaskResult{ TaskID: asyncTask.TaskID, TaskType: string(asyncTask.TaskType), @@ -99,7 +74,6 @@ func AsyncTaskResultDetailHandler(c *gin.Context) { Progress: asyncTask.Progress, } - // Add result or error information if available if taskResult != nil { if taskResult.Result != nil { responseTask.Result = map[string]any(taskResult.Result) @@ -115,10 +89,5 @@ func AsyncTaskResultDetailHandler(c *gin.Context) { } } - // Return success response - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: 2000, - Msg: "query completed", - Payload: responseTask, - }) + renderRespSuccess(c, constants.RespCodeSuccess, "query completed", responseTask) } diff --git a/handler/async_task_result_query_handler.go b/handler/async_task_result_query_handler.go index 12c7d67..e9b08ae 100644 --- a/handler/async_task_result_query_handler.go +++ b/handler/async_task_result_query_handler.go @@ -2,9 +2,9 @@ package handler import ( - "net/http" "strings" + "modelRT/constants" "modelRT/database" "modelRT/logger" "modelRT/network" @@ -22,34 +22,25 @@ import ( // @Produce json // @Param task_ids query string true "任务ID列表，用逗号分隔" // @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskResultQueryResponse} "查询成功" -// @Failure 400 {object} network.FailureResponse "请求参数错误" -// @Failure 500 {object} network.FailureResponse "服务器内部错误" +// @Failure 200 {object} network.FailureResponse "请求参数错误" // @Router /task/async/results [get] func AsyncTaskResultQueryHandler(c *gin.Context) { ctx := c.Request.Context() - // Parse task IDs from query parameter taskIDsParam := c.Query("task_ids") if taskIDsParam == "" { logger.Error(ctx, "task_ids parameter is required") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "task_ids parameter is required", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "task_ids parameter is required", nil) return } - // Parse comma-separated task IDs var taskIDs []uuid.UUID taskIDStrs := splitCommaSeparated(taskIDsParam) for _, taskIDStr := range taskIDStrs { taskID, err := uuid.FromString(taskIDStr) if err != nil { logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid task ID format", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid task ID format", nil) return } taskIDs = append(taskIDs, taskID) @@ -57,52 +48,36 @@ func AsyncTaskResultQueryHandler(c *gin.Context) { if len(taskIDs) == 0 { logger.Error(ctx, "no valid task IDs provided") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "no valid task IDs provided", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "no valid task IDs provided", nil) return } pgClient := database.GetPostgresDBClient() if pgClient == nil { logger.Error(ctx, "database connection not found in context") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "database connection error", - }) + renderRespFailure(c, constants.RespCodeServerError, "database connection error", nil) return } - // Query tasks from database asyncTasks, err := database.GetAsyncTasksByIDs(ctx, pgClient, taskIDs) if err != nil { logger.Error(ctx, "failed to query async tasks from database", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to query tasks", - }) + renderRespFailure(c, constants.RespCodeServerError, "failed to query tasks", nil) return } - // Query task results from database taskResults, err := database.GetAsyncTaskResults(ctx, pgClient, taskIDs) if err != nil { logger.Error(ctx, "failed to query async task results from database", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to query task results", - }) + renderRespFailure(c, constants.RespCodeServerError, "failed to query task results", nil) return } - // Create a map of task results for easy lookup taskResultMap := make(map[uuid.UUID]orm.AsyncTaskResult) for _, result := range taskResults { taskResultMap[result.TaskID] = result } - // Convert to response format var responseTasks []network.AsyncTaskResult for _, asyncTask := range asyncTasks { taskResult := network.AsyncTaskResult{ @@ -114,7 +89,6 @@ func AsyncTaskResultQueryHandler(c *gin.Context) { Progress: asyncTask.Progress, } - // Add result or error information if available if result, exists := taskResultMap[asyncTask.TaskID]; exists { if result.Result != nil { taskResult.Result = map[string]any(result.Result) @@ -133,14 +107,9 @@ func AsyncTaskResultQueryHandler(c *gin.Context) { responseTasks = append(responseTasks, taskResult) } - // Return success response - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: 2000, - Msg: "query completed", - Payload: network.AsyncTaskResultQueryResponse{ - Total: len(responseTasks), - Tasks: responseTasks, - }, + renderRespSuccess(c, constants.RespCodeSuccess, "query completed", network.AsyncTaskResultQueryResponse{ + Total: len(responseTasks), + Tasks: responseTasks, }) } diff --git a/handler/async_task_status_update_handler.go b/handler/async_task_status_update_handler.go index caeab2b..daf3c72 100644 --- a/handler/async_task_status_update_handler.go +++ b/handler/async_task_status_update_handler.go @@ -2,8 +2,7 @@ package handler import ( - "net/http" - + "modelRT/constants" "modelRT/database" "modelRT/logger" "modelRT/network" @@ -19,14 +18,10 @@ func AsyncTaskStatusUpdateHandler(c *gin.Context) { if err := c.ShouldBindJSON(&request); err != nil { logger.Error(ctx, "failed to unmarshal async task status update request", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid request parameters", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid request parameters", nil) return } - // Validate status validStatus := map[string]bool{ string(orm.AsyncTaskStatusSubmitted): true, string(orm.AsyncTaskStatusRunning): true, @@ -36,36 +31,25 @@ func AsyncTaskStatusUpdateHandler(c *gin.Context) { if !validStatus[request.Status] { logger.Error(ctx, "invalid task status", "status", request.Status) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: "invalid task status", - }) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid task status", nil) return } pgClient := database.GetPostgresDBClient() if pgClient == nil { logger.Error(ctx, "database connection not found in context") - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "database connection error", - }) + renderRespFailure(c, constants.RespCodeServerError, "database connection error", nil) return } - // Update task status status := orm.AsyncTaskStatus(request.Status) err := database.UpdateAsyncTaskStatus(ctx, pgClient, request.TaskID, status) if err != nil { logger.Error(ctx, "failed to update async task status", "task_id", request.TaskID, "status", request.Status, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to update task status", - }) + renderRespFailure(c, constants.RespCodeServerError, "failed to update task status", nil) return } - // If task is completed or failed, update finished_at timestamp if request.Status == string(orm.AsyncTaskStatusCompleted) { err = database.CompleteAsyncTask(ctx, pgClient, request.TaskID, request.Timestamp) } else if request.Status == string(orm.AsyncTaskStatusFailed) { @@ -74,16 +58,9 @@ func AsyncTaskStatusUpdateHandler(c *gin.Context) { if err != nil { logger.Error(ctx, "failed to update async task completion timestamp", "task_id", request.TaskID, "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusInternalServerError, - Msg: "failed to update task completion timestamp", - }) + renderRespFailure(c, constants.RespCodeServerError, "failed to update task completion timestamp", nil) return } - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: 2000, - Msg: "task status updated successfully", - Payload: nil, - }) + renderRespSuccess(c, constants.RespCodeSuccess, "task status updated successfully", nil) } diff --git a/task/base_task.go b/task/base_task.go index 271d043..c49b897 100644 --- a/task/base_task.go +++ b/task/base_task.go @@ -9,8 +9,8 @@ import ( "gorm.io/gorm" ) -// TaskParams defines the interface for task-specific parameters -type TaskParams interface { +// Params defines the interface for task-specific parameters +type Params interface { Validate() error GetType() UnifiedTaskType ToMap() map[string]interface{} @@ -20,12 +20,12 @@ type TaskParams interface { // BaseTask provides common functionality for all task implementations type BaseTask struct { taskType UnifiedTaskType - params TaskParams + params Params name string } // NewBaseTask creates a new BaseTask instance -func NewBaseTask(taskType UnifiedTaskType, params TaskParams, name string) *BaseTask { +func NewBaseTask(taskType UnifiedTaskType, params Params, name string) *BaseTask { return &BaseTask{ taskType: taskType, params: params, @@ -37,7 +37,7 @@ func (t *BaseTask) GetType() UnifiedTaskType { return t.taskType } -func (t *BaseTask) GetParams() TaskParams { +func (t *BaseTask) GetParams() Params { return t.params } diff --git a/task/handler_factory.go b/task/handler_factory.go index 7fcc174..b705e27 100644 --- a/task/handler_factory.go +++ b/task/handler_factory.go @@ -158,7 +158,6 @@ func (h *TopologyAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, // check the start node itself before BFS if !inServiceMap[startComponentUUID] { - fmt.Println(11111) return persistTopologyResult(ctx, db, taskID, startComponentUUID, endComponentUUID, checkInService, false, nil, &startComponentUUID) } @@ -221,7 +220,6 @@ func (h *TopologyAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, // parseTopologyAnalysisParams extracts and validates the three required fields. // check_in_service defaults to true when absent. func parseTopologyAnalysisParams(params map[string]any) (startID, endID uuid.UUID, checkInService bool, err error) { - fmt.Printf("params:%+v\n", params) startStr, ok := params["start_component_uuid"].(string) if !ok || startStr == "" { err = fmt.Errorf("missing or invalid start_component_uuid") From 1ee722dd589c6e385c188520b17386c540f23ae1 Mon Sep 17 00:00:00 2001 From: douxu Date: Thu, 7 May 2026 16:43:34 +0800 Subject: [PATCH 34/43] refactor: migrate trace propagation from B3 to W3C TraceContext - switch OTel propagator from b3.New() to propagation.TraceContext{} - rename B3 header constants to generic internal context keys - remove go.opentelemetry.io/contrib/propagators/b3 dependency - add amqpHeaderCarrier to inject W3C traceparent into AMQP message headers --- constants/trace.go | 10 ++++++---- go.mod | 1 - go.sum | 2 -- middleware/trace.go | 10 +++------- mq/publish_up_down_limit_event.go | 33 +++++++++++++++++++++++++++++++ 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/constants/trace.go b/constants/trace.go index ab5c5df..e5e595a 100644 --- a/constants/trace.go +++ b/constants/trace.go @@ -1,11 +1,13 @@ // Package constants define constant variable package constants -// Assuming the B3 specification +// Internal context keys for trace values set by StartTrace middleware. +// These are gin/stdlib context keys only — actual W3C header propagation +// (traceparent / tracestate) is handled automatically by the OTel propagator. const ( - HeaderTraceID = "X-B3-TraceId" - HeaderSpanID = "X-B3-SpanId" - HeaderParentSpanID = "X-B3-ParentSpanId" + HeaderTraceID = "trace-id" + HeaderSpanID = "span-id" + HeaderParentSpanID = "parent-span-id" ) // traceCtxKey is an unexported type for context keys to avoid collisions with other packages. diff --git a/go.mod b/go.mod index a815516..9f24f78 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.4 github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 - go.opentelemetry.io/contrib/propagators/b3 v1.43.0 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 go.opentelemetry.io/otel/sdk v1.43.0 diff --git a/go.sum b/go.sum index 97e2fcf..19c0d18 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,6 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfS github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/propagators/b3 v1.43.0 h1:CETqV3QLLPTy5yNrqyMr41VnAOOD4lsRved7n4QG00A= -go.opentelemetry.io/contrib/propagators/b3 v1.43.0/go.mod h1:Q4mCiCdziYzpNR0g+6UqVotAlCDZdzz6L8jwY4knOrw= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= diff --git a/middleware/trace.go b/middleware/trace.go index 9712de8..65c1ee2 100644 --- a/middleware/trace.go +++ b/middleware/trace.go @@ -14,7 +14,6 @@ import ( "modelRT/logger" "github.com/gin-gonic/gin" - "go.opentelemetry.io/contrib/propagators/b3" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" @@ -25,7 +24,7 @@ import ( ) // InitTracerProvider creates an OTLP TracerProvider and registers it as the global provider. -// It also registers the B3 propagator to stay compatible with existing B3 infrastructure. +// It registers the W3C TraceContext propagator (traceparent header). // The caller is responsible for calling Shutdown on the returned provider during graceful shutdown. func InitTracerProvider(ctx context.Context, cfg config.ModelRTConfig) (*sdktrace.TracerProvider, error) { opts := []otlptracehttp.Option{ @@ -52,18 +51,15 @@ func InitTracerProvider(ctx context.Context, cfg config.ModelRTConfig) (*sdktrac ) otel.SetTracerProvider(tp) - otel.SetTextMapPropagator(b3.New()) + otel.SetTextMapPropagator(propagation.TraceContext{}) return tp, nil } -// StartTrace extracts upstream B3 trace context from request headers and starts a server span. -// Typed context keys are also injected for backward compatibility with the existing logger -// until the logger is migrated to read from the OTel span context (Step 6). +// StartTrace extracts upstream W3C trace context from request headers and starts a server span. func StartTrace() gin.HandlerFunc { tracer := otel.Tracer("modelRT/http") return func(c *gin.Context) { - // Extract upstream trace context from B3 headers (X-B3-TraceId etc.) ctx := otel.GetTextMapPropagator().Extract( c.Request.Context(), propagation.HeaderCarrier(c.Request.Header), diff --git a/mq/publish_up_down_limit_event.go b/mq/publish_up_down_limit_event.go index 9d4bbbf..65879d7 100644 --- a/mq/publish_up_down_limit_event.go +++ b/mq/publish_up_down_limit_event.go @@ -11,8 +11,36 @@ import ( "modelRT/mq/event" amqp "github.com/rabbitmq/amqp091-go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" ) +// amqpHeaderCarrier adapts amqp.Table to propagation.TextMapCarrier for trace context injection. +type amqpHeaderCarrier amqp.Table + +func (c amqpHeaderCarrier) Get(key string) string { + val, ok := amqp.Table(c)[key] + if !ok { + return "" + } + str, _ := val.(string) + return str +} + +func (c amqpHeaderCarrier) Set(key, value string) { + amqp.Table(c)[key] = value +} + +func (c amqpHeaderCarrier) Keys() []string { + keys := make([]string, 0, len(c)) + for k := range c { + keys = append(keys, k) + } + return keys +} + +var _ propagation.TextMapCarrier = amqpHeaderCarrier{} + // MsgChan define variable of channel to store messages that need to be sent to rabbitMQ var MsgChan chan *event.EventRecord @@ -124,6 +152,10 @@ func PushUpDownLimitEventToRabbitMQ(ctx context.Context, msgChan chan *event.Eve continue } + // inject current trace context into AMQP headers so eventRT can restore the span chain + headers := amqp.Table{} + otel.GetTextMapPropagator().Inject(ctx, amqpHeaderCarrier(headers)) + // send event alarm message to rabbitMQ queue routingKey := eventRecord.Category pubCtx, cancel := context.WithTimeout(ctx, 5*time.Second) @@ -135,6 +167,7 @@ func PushUpDownLimitEventToRabbitMQ(ctx context.Context, msgChan chan *event.Eve amqp.Publishing{ ContentType: "text/plain", Body: recordBytes, + Headers: headers, }) cancel() From 1dd8491440cf584a0589ca881cc4544aba7e40cf Mon Sep 17 00:00:00 2001 From: douxu Date: Fri, 8 May 2026 16:19:12 +0800 Subject: [PATCH 35/43] refactor: replace EventStatusPersisted with IsPersisted field on EventRecord - add IsPersisted bool to EventRecord for explicit persistence tracking by eventRT consumer - remove EventStatusPersisted constant, decoupling DB persistence from event lifecycle status - update event status comments for accuracy and CIM-agnostic language --- constants/event.go | 10 ++++------ mq/event/event.go | 2 ++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/constants/event.go b/constants/event.go index e4ea62e..11dbf29 100644 --- a/constants/event.go +++ b/constants/event.go @@ -52,15 +52,13 @@ const ( const ( // EventStatusHappended define status for event record when event just happened, no data attached yet EventStatusHappended = iota - // EventStatusDataAttached define status for event record when event just happened, data attached already + // EventStatusDataAttached define status for event record when event data attached, ready to be sent EventStatusDataAttached - // EventStatusReported define status for event record when event reported to CIM, no matter it's successful or failed + // EventStatusReported define status for event record when event reported to downstream, no matter it's successful or failed EventStatusReported - // EventStatusConfirmed define status for event record when event confirmed by CIM, no matter it's successful or failed + // EventStatusConfirmed define status for event record when event confirmed by operator or CIM EventStatusConfirmed - // EventStatusPersisted define status for event record when event persisted in database, no matter it's successful or failed - EventStatusPersisted - // EventStatusClosed define status for event record when event closed, no matter it's successful or failed + // EventStatusClosed define status for event record when event closed due to condition recovery or manual close EventStatusClosed ) diff --git a/mq/event/event.go b/mq/event/event.go index 6740251..951242e 100644 --- a/mq/event/event.go +++ b/mq/event/event.go @@ -13,6 +13,8 @@ type EventRecord struct { Priority int `json:"priority"` // 事件状态 Status int `json:"status"` + // 是否已持久化到数据库，由 eventRT 消费并落库后置为 true + IsPersisted bool `json:"is_persisted"` // 可选模板参数 Category string `json:"category,omitempty"` // 毫秒级时间戳 (Unix epoch) From cccd4becdceb631573b1317fcc8d1a62548ba44f Mon Sep 17 00:00:00 2001 From: douxu Date: Mon, 11 May 2026 17:34:27 +0800 Subject: [PATCH 36/43] feat: add Loki logging, fix MQ shutdown order, improve realtime tracing - add LokiConfig and batching lokiSyncer for dev-mode direct log push - refactor zap logger to support mode-aware encoding and K8s pod fields - fix RabbitMQ shutdown race: move CloseRabbitProxy to defer so channel closes before connection (prevents 504 error on Ctrl+C) - wrap MsgChan with EventMessage to carry per-cycle trace carrier - create new root OTel span per computation cycle linked to startup span, giving each cycle an independent traceID with startup as reference --- config/config.go | 21 ++- logger/loki_syncer.go | 133 ++++++++++++++++++ logger/zap.go | 96 +++++++++---- main.go | 3 +- mq/publish_up_down_limit_event.go | 21 ++- real-time-data/compute_analyzer.go | 12 +- .../real_time_data_up_down_limit_computing.go | 30 +++- task/worker.go | 4 +- 8 files changed, 266 insertions(+), 54 deletions(-) create mode 100644 logger/loki_syncer.go diff --git a/config/config.go b/config/config.go index c4a69d7..5a87f34 100644 --- a/config/config.go +++ b/config/config.go @@ -56,15 +56,22 @@ type PostgresConfig struct { Password string `mapstructure:"password"` } +// LokiConfig define config struct of loki direct-push (used in development mode) +type LokiConfig struct { + Endpoint string `mapstructure:"endpoint"` // empty disables direct push + Labels map[string]string `mapstructure:"labels"` +} + // LoggerConfig define config struct of zap logger config type LoggerConfig struct { - Mode string `mapstructure:"mode"` - Level string `mapstructure:"level"` - FilePath string `mapstructure:"filepath"` - MaxSize int `mapstructure:"maxsize"` - MaxBackups int `mapstructure:"maxbackups"` - MaxAge int `mapstructure:"maxage"` - Compress bool `mapstructure:"compress"` + Mode string `mapstructure:"mode"` + Level string `mapstructure:"level"` + FilePath string `mapstructure:"filepath"` // empty disables file rotation in container modes + MaxSize int `mapstructure:"maxsize"` + MaxBackups int `mapstructure:"maxbackups"` + MaxAge int `mapstructure:"maxage"` + Compress bool `mapstructure:"compress"` + Loki LokiConfig `mapstructure:"loki"` } // RedisConfig define config struct of redis config diff --git a/logger/loki_syncer.go b/logger/loki_syncer.go new file mode 100644 index 0000000..3e3f319 --- /dev/null +++ b/logger/loki_syncer.go @@ -0,0 +1,133 @@ +// Package logger define log struct of modelRT project +package logger + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "strconv" + "sync" + "time" + + "modelRT/config" +) + +type lokiPushRequest struct { + Streams []lokiStream `json:"streams"` +} + +type lokiStream struct { + Stream map[string]string `json:"stream"` + Values [][2]string `json:"values"` +} + +// lokiSyncer implements zapcore.WriteSyncer, batching log lines and pushing them +// to Loki's push API asynchronously. Errors are silently dropped so a unreachable +// Loki instance never blocks or crashes the application. +type lokiSyncer struct { + endpoint string + labels map[string]string + client *http.Client + ch chan string + wg sync.WaitGroup + closeOnce sync.Once +} + +func newLokiSyncer(lCfg config.LokiConfig) *lokiSyncer { + // always tag development logs with env=development; caller-supplied labels override if needed + labels := map[string]string{"env": "development"} + for k, v := range lCfg.Labels { + labels[k] = v + } + ls := &lokiSyncer{ + endpoint: lCfg.Endpoint + "/loki/api/v1/push", + labels: labels, + client: &http.Client{Timeout: 5 * time.Second}, + ch: make(chan string, 512), + } + ls.wg.Add(1) + go ls.run() + return ls +} + +func (ls *lokiSyncer) Write(p []byte) (int, error) { + select { + case ls.ch <- string(p): + default: + // channel full: drop the line rather than block the caller + } + return len(p), nil +} + +// Sync flushes remaining buffered lines and shuts down the background goroutine. +// Called by zap.Logger.Sync() at application shutdown. +func (ls *lokiSyncer) Sync() error { + ls.closeOnce.Do(func() { close(ls.ch) }) + ls.wg.Wait() + return nil +} + +func (ls *lokiSyncer) run() { + defer ls.wg.Done() + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + var batch []string + flush := func() { + if len(batch) == 0 { + return + } + ls.push(batch) + batch = batch[:0] + } + + for { + select { + case line, ok := <-ls.ch: + if !ok { + flush() + return + } + batch = append(batch, line) + if len(batch) >= 100 { + flush() + } + case <-ticker.C: + flush() + } + } +} + +func (ls *lokiSyncer) push(lines []string) { + ts := strconv.FormatInt(time.Now().UnixNano(), 10) + values := make([][2]string, len(lines)) + for i, line := range lines { + values[i] = [2]string{ts, line} + } + + body, err := json.Marshal(lokiPushRequest{ + Streams: []lokiStream{{Stream: ls.labels, Values: values}}, + }) + if err != nil { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, ls.endpoint, bytes.NewReader(body)) + if err != nil { + return + } + req.Header.Set("Content-Type", "application/json") + + resp, err := ls.client.Do(req) + if err != nil { + fmt.Fprintf(os.Stderr, "loki syncer: push failed: %v\n", err) + return + } + defer resp.Body.Close() +} diff --git a/logger/zap.go b/logger/zap.go index 5e65db7..52d74eb 100644 --- a/logger/zap.go +++ b/logger/zap.go @@ -21,53 +21,89 @@ var ( _globalLogger *zap.Logger ) -// getEncoder responsible for setting the log format for encoding -func getEncoder() zapcore.Encoder { - encoderConfig := zap.NewProductionEncoderConfig() - // serialization time eg:2006-01-02 15:04:05 - encoderConfig.EncodeTime = zapcore.TimeEncoderOfLayout("2006-01-02 15:04:05") - encoderConfig.TimeKey = "time" - encoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder - encoderConfig.EncodeCaller = zapcore.ShortCallerEncoder - return zapcore.NewJSONEncoder(encoderConfig) +// getEncoder returns a console encoder for development (human-readable, colored) and a JSON encoder +// for container modes (parseable by Promtail pipeline_stages). +func getEncoder(mode string) zapcore.Encoder { + cfg := zap.NewProductionEncoderConfig() + cfg.EncodeTime = zapcore.TimeEncoderOfLayout("2006-01-02 15:04:05") + cfg.TimeKey = "time" + cfg.EncodeCaller = zapcore.ShortCallerEncoder + if mode == constants.DevelopmentLogMode { + cfg.EncodeLevel = zapcore.CapitalColorLevelEncoder + return zapcore.NewConsoleEncoder(cfg) + } + cfg.EncodeLevel = zapcore.CapitalLevelEncoder + return zapcore.NewJSONEncoder(cfg) } -// getLogWriter responsible for setting the location of log storage -func getLogWriter(mode, filename string, maxsize, maxBackup, maxAge int, compress bool) zapcore.WriteSyncer { - dateStr := time.Now().Format("2006-01-02 15:04:05") - finalFilename := fmt.Sprintf(filename, dateStr) - lumberJackLogger := &lumberjack.Logger{ - Filename: finalFilename, // log file position - MaxSize: maxsize, // log file maxsize - MaxAge: maxAge, // maximum number of day files retained - MaxBackups: maxBackup, // maximum number of old files retained - Compress: compress, // whether to compress +// getWriteSyncer returns write targets based on mode: +// - development: stdout + optional Loki direct-push (when loki.endpoint is set) +// - container modes: stdout always (Promtail collects) + rotating file (when filepath is set) +func getWriteSyncer(lCfg config.LoggerConfig) zapcore.WriteSyncer { + stdout := zapcore.AddSync(os.Stdout) + + if lCfg.Mode == constants.DevelopmentLogMode { + if lCfg.Loki.Endpoint == "" { + return stdout + } + return zapcore.NewMultiWriteSyncer(stdout, newLokiSyncer(lCfg.Loki)) } - syncConsole := zapcore.AddSync(os.Stderr) - if mode == constants.DevelopmentLogMode { - return syncConsole + syncers := []zapcore.WriteSyncer{stdout} + if lCfg.FilePath != "" { + dateStr := time.Now().Format("2006-01-02 15:04:05") + syncers = append(syncers, zapcore.AddSync(&lumberjack.Logger{ + Filename: fmt.Sprintf(lCfg.FilePath, dateStr), + MaxSize: lCfg.MaxSize, + MaxAge: lCfg.MaxAge, + MaxBackups: lCfg.MaxBackups, + Compress: lCfg.Compress, + })) } + return zapcore.NewMultiWriteSyncer(syncers...) +} - syncFile := zapcore.AddSync(lumberJackLogger) - return zapcore.NewMultiWriteSyncer(syncFile, syncConsole) +// containerFields reads K8s Downward API environment variables and returns them as global zap fields. +// These fields appear on every log line, allowing Loki/Grafana to filter by pod, namespace, and node. +// Inject them in the Deployment manifest: +// +// env: +// - name: K8S_NAMESPACE +// valueFrom: {fieldRef: {fieldPath: metadata.namespace}} +// - name: K8S_NODE_NAME +// valueFrom: {fieldRef: {fieldPath: spec.nodeName}} +func containerFields() []zap.Field { + var fields []zap.Field + // HOSTNAME is automatically set to the pod name by Kubernetes. + if pod := os.Getenv("HOSTNAME"); pod != "" { + fields = append(fields, zap.String("pod", pod)) + } + if ns := os.Getenv("K8S_NAMESPACE"); ns != "" { + fields = append(fields, zap.String("namespace", ns)) + } + if node := os.Getenv("K8S_NODE_NAME"); node != "" { + fields = append(fields, zap.String("node", node)) + } + return fields } // initLogger return successfully initialized zap logger func initLogger(lCfg config.LoggerConfig) *zap.Logger { - writeSyncer := getLogWriter(lCfg.Mode, lCfg.FilePath, lCfg.MaxSize, lCfg.MaxBackups, lCfg.MaxAge, lCfg.Compress) - encoder := getEncoder() + writeSyncer := getWriteSyncer(lCfg) + encoder := getEncoder(lCfg.Mode) l := new(zapcore.Level) - err := l.UnmarshalText([]byte(lCfg.Level)) - if err != nil { + if err := l.UnmarshalText([]byte(lCfg.Level)); err != nil { panic(err) } core := zapcore.NewCore(encoder, writeSyncer, l) - logger := zap.New(core, zap.AddCaller()) + opts := []zap.Option{zap.AddCaller()} + if lCfg.Mode != constants.DevelopmentLogMode { + opts = append(opts, zap.Fields(containerFields()...)) + } + logger := zap.New(core, opts...) - // 替换全局日志实例 zap.ReplaceGlobals(logger) return logger } diff --git a/main.go b/main.go index 4121a1f..f120c6f 100644 --- a/main.go +++ b/main.go @@ -174,6 +174,7 @@ func main() { // init rabbitmq connection mq.InitRabbitProxy(ctx, modelRTConfig.RabbitMQConfig) + defer mq.CloseRabbitProxy() // init async task worker taskWorker, err := task.InitTaskWorker(ctx, modelRTConfig, postgresDBClient) @@ -284,7 +285,6 @@ func main() { if err := server.Shutdown(context.Background()); err != nil { logger.Error(ctx, "shutdown serverError", "err", err) } - mq.CloseRabbitProxy() logger.Info(ctx, "resources cleaned up, exiting") }() @@ -300,4 +300,3 @@ func main() { } } } - diff --git a/mq/publish_up_down_limit_event.go b/mq/publish_up_down_limit_event.go index 65879d7..07a122b 100644 --- a/mq/publish_up_down_limit_event.go +++ b/mq/publish_up_down_limit_event.go @@ -41,11 +41,17 @@ func (c amqpHeaderCarrier) Keys() []string { var _ propagation.TextMapCarrier = amqpHeaderCarrier{} +// EventMessage wraps an EventRecord with the trace context of the computation cycle that produced it. +type EventMessage struct { + Record *event.EventRecord + TraceCarrier map[string]string +} + // MsgChan define variable of channel to store messages that need to be sent to rabbitMQ -var MsgChan chan *event.EventRecord +var MsgChan chan *EventMessage func init() { - MsgChan = make(chan *event.EventRecord, 10000) + MsgChan = make(chan *EventMessage, 10000) } func initUpDownLimitEventChannel(ctx context.Context) (*amqp.Channel, error) { @@ -107,7 +113,7 @@ func initUpDownLimitEventChannel(ctx context.Context) (*amqp.Channel, error) { } // PushUpDownLimitEventToRabbitMQ define func to push up and down limit event message to rabbitMQ -func PushUpDownLimitEventToRabbitMQ(ctx context.Context, msgChan chan *event.EventRecord) { +func PushUpDownLimitEventToRabbitMQ(ctx context.Context, msgChan chan *EventMessage) { channel, err := initUpDownLimitEventChannel(ctx) if err != nil { logger.Error(ctx, "initializing rabbitMQ channel failed", "error", err) @@ -138,13 +144,15 @@ func PushUpDownLimitEventToRabbitMQ(ctx context.Context, msgChan chan *event.Eve logger.Info(ctx, "push event alarm message to rabbitMQ stopped by context cancel") channel.Close() return - case eventRecord, ok := <-msgChan: + case msg, ok := <-msgChan: if !ok { logger.Info(ctx, "push event alarm message to rabbitMQ stopped by msgChan closed, exiting push loop") channel.Close() return } + eventRecord := msg.Record + // TODO 将消息的序列化移动到发送之前，以便使用eventRecord的category来作为routing key recordBytes, err := json.Marshal(eventRecord) if err != nil { @@ -152,9 +160,10 @@ func PushUpDownLimitEventToRabbitMQ(ctx context.Context, msgChan chan *event.Eve continue } - // inject current trace context into AMQP headers so eventRT can restore the span chain + // restore computation cycle trace context so the AMQP message carries the correct parent span + msgCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(msg.TraceCarrier)) headers := amqp.Table{} - otel.GetTextMapPropagator().Inject(ctx, amqpHeaderCarrier(headers)) + otel.GetTextMapPropagator().Inject(msgCtx, amqpHeaderCarrier(headers)) // send event alarm message to rabbitMQ queue routingKey := eventRecord.Category diff --git a/real-time-data/compute_analyzer.go b/real-time-data/compute_analyzer.go index d65a343..864fd2d 100644 --- a/real-time-data/compute_analyzer.go +++ b/real-time-data/compute_analyzer.go @@ -11,6 +11,9 @@ import ( "modelRT/logger" "modelRT/mq" "modelRT/mq/event" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" ) // RealTimeAnalyzer define interface general methods for real-time data analysis and event triggering @@ -146,6 +149,9 @@ func analyzeTEDataLogic(ctx context.Context, conf *ComputeConfig, thresholds teE } } + carrier := make(map[string]string) + otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(carrier)) + for breachType, trigger := range breachTriggers { // trigger Action command, mainBody := genTEEventCommandAndMainBody(ctx, conf.Action) @@ -155,7 +161,7 @@ func analyzeTEDataLogic(ctx context.Context, conf *ComputeConfig, thresholds teE logger.Error(ctx, "trigger event action failed", "error", err) return } - mq.MsgChan <- eventRecord + mq.MsgChan <- &mq.EventMessage{Record: eventRecord, TraceCarrier: carrier} } } @@ -330,7 +336,9 @@ func analyzeTIDataLogic(ctx context.Context, conf *ComputeConfig, thresholds tiE logger.Error(ctx, "trigger event action failed", "error", err) return } - mq.MsgChan <- eventRecord + carrier := make(map[string]string) + otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(carrier)) + mq.MsgChan <- &mq.EventMessage{Record: eventRecord, TraceCarrier: carrier} return } } diff --git a/real-time-data/real_time_data_up_down_limit_computing.go b/real-time-data/real_time_data_up_down_limit_computing.go index 53acc8d..24a93e9 100644 --- a/real-time-data/real_time_data_up_down_limit_computing.go +++ b/real-time-data/real_time_data_up_down_limit_computing.go @@ -14,6 +14,10 @@ import ( "modelRT/network" "modelRT/orm" "modelRT/util" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + oteltrace "go.opentelemetry.io/otel/trace" ) var ( @@ -205,25 +209,41 @@ func continuousComputation(ctx context.Context, conf *ComputeConfig) { logger.Info(ctx, "continuous computing goroutine stopped by parent context done signal") return case <-ticker.C: - queryCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + // start a new root span for this computation cycle, linked to the startup span + startupSpanCtx := oteltrace.SpanFromContext(ctx).SpanContext() + cycleCtx, cycleSpan := otel.Tracer("modelRT/realtime").Start( + context.Background(), + "realtime.compute_cycle", + oteltrace.WithNewRoot(), + oteltrace.WithLinks(oteltrace.Link{SpanContext: startupSpanCtx}), + oteltrace.WithAttributes( + attribute.String("measurement_uuid", uuid), + attribute.String("query_key", conf.QueryKey), + ), + ) + + queryCtx, cancel := context.WithTimeout(cycleCtx, 2*time.Second) members, err := client.QueryByZRange(queryCtx, conf.QueryKey, conf.DataSize) cancel() if err != nil { - logger.Error(ctx, "query real time data from redis failed", "key", conf.QueryKey, "error", err) + logger.Error(cycleCtx, "query real time data from redis failed", "key", conf.QueryKey, "error", err) + cycleSpan.End() continue } realTimedatas := util.ConvertZSetMembersToFloat64(members) if len(realTimedatas) == 0 { - logger.Info(ctx, "no real time data queried from redis, skip this computation cycle", "key", conf.QueryKey) + logger.Info(cycleCtx, "no real time data queried from redis, skip this computation cycle", "key", conf.QueryKey) + cycleSpan.End() continue } if conf.Analyzer != nil { - conf.Analyzer.AnalyzeAndTriggerEvent(ctx, conf, realTimedatas) + conf.Analyzer.AnalyzeAndTriggerEvent(cycleCtx, conf, realTimedatas) } else { - logger.Error(ctx, "analyzer is not initialized for this measurement", "uuid", uuid) + logger.Error(cycleCtx, "analyzer is not initialized for this measurement", "uuid", uuid) } + cycleSpan.End() } } } diff --git a/task/worker.go b/task/worker.go index 20795cf..b51e00d 100644 --- a/task/worker.go +++ b/task/worker.go @@ -537,11 +537,11 @@ func (w *TaskWorker) Stop() error { // Close channel if w.ch != nil { if err := w.ch.Close(); err != nil { - logger.Error(w.ctx, "Failed to close channel", "error", err) + logger.Error(w.ctx, "failed to close channel", "error", err) } } - logger.Info(w.ctx, "Task worker stopped") + logger.Info(w.ctx, "task worker stopped") return nil } From 42956d17935429df5560ef2921fe38d7d5624ee4 Mon Sep 17 00:00:00 2001 From: douxu Date: Wed, 13 May 2026 16:58:36 +0800 Subject: [PATCH 37/43] feat: add dedicated message-exchange for task lifecycle notifications - add constants/message.go with MessageTask* categories and message-exchange / message-queue / dead-letter routing constants - add mq/publish_message.go with PushMessageToRabbitMQ (confirm mode, dead-letter queue) separate from the existing event-exchange publisher - add mq/emit.go with TryEmitMessage for non-blocking, OTel-traced dispatch - add mq/event/task_event_gen.go with NewTaskSubmitted/Running/Completed/ Failed/CancelledMessage constructors - wire TryEmitMessage into task worker and create/cancel handlers so all 5 lifecycle transitions are published (previously task.* routed to event-exchange with no matching binding, causing silent drops) - harden Dockerfile: scratch final image, pinned alpine:3.21 certs stage, apk upgrade in builder, add -trimpath -mod=readonly go build flags - add full K8s manifests under deploy/k8s/ for Redis, RabbitMQ (mTLS), ModelRT (Downward API, scratch image, readOnlyRootFilesystem), Jaeger, Loki, Promtail, Grafana - expand deploy.md with async_task SQL schema, TLS cert generation steps, K8s deployment procedures, and SSH tunnel configuration --- constants/event.go | 7 + constants/message.go | 33 ++ deploy/deploy.md | 506 +++++++++++++++++- deploy/dockerfile/modelrt.Dockerfile | 34 +- deploy/k8s/grafana-configmap.yaml | 26 + deploy/k8s/grafana-deployment.yaml | 41 ++ deploy/k8s/grafana-service.yaml | 14 + .../jaeger-deployment.yaml} | 28 - deploy/k8s/jaeger-service.yaml | 27 + deploy/k8s/loki-configmap.yaml | 49 ++ deploy/k8s/loki-deployment.yaml | 45 ++ deploy/k8s/loki-pvc.yaml | 11 + deploy/k8s/loki-service.yaml | 14 + deploy/k8s/modelrt-certs-secret.sh | 14 + deploy/k8s/modelrt-configmap.yaml | 86 +++ deploy/k8s/modelrt-deployment.yaml | 90 ++++ deploy/k8s/modelrt-secret.yaml | 8 + deploy/k8s/modelrt-service.yaml | 15 + deploy/k8s/promtail-configmap.yaml | 52 ++ deploy/k8s/promtail-daemonset.yaml | 51 ++ deploy/k8s/promtail-rbac.yaml | 27 + deploy/k8s/rabbitmq-config.yaml | 33 ++ deploy/k8s/rabbitmq-deployment.yaml | 81 +++ deploy/k8s/rabbitmq-secret.yaml | 9 + deploy/k8s/rabbitmq-service.yaml | 29 + deploy/k8s/rabbitmq-users-config.yaml | 77 +++ deploy/k8s/redis-deployment.yaml | 23 + deploy/k8s/redis-service.yaml | 13 + handler/async_task_cancel_handler.go | 6 + handler/async_task_create_handler.go | 6 + main.go | 2 + mq/emit.go | 28 + mq/event/task_event_gen.go | 81 +++ mq/publish_message.go | 149 ++++++ task/worker.go | 13 + 35 files changed, 1688 insertions(+), 40 deletions(-) create mode 100644 constants/message.go create mode 100644 deploy/k8s/grafana-configmap.yaml create mode 100644 deploy/k8s/grafana-deployment.yaml create mode 100644 deploy/k8s/grafana-service.yaml rename deploy/{jaeger.yaml => k8s/jaeger-deployment.yaml} (52%) create mode 100644 deploy/k8s/jaeger-service.yaml create mode 100644 deploy/k8s/loki-configmap.yaml create mode 100644 deploy/k8s/loki-deployment.yaml create mode 100644 deploy/k8s/loki-pvc.yaml create mode 100644 deploy/k8s/loki-service.yaml create mode 100644 deploy/k8s/modelrt-certs-secret.sh create mode 100644 deploy/k8s/modelrt-configmap.yaml create mode 100644 deploy/k8s/modelrt-deployment.yaml create mode 100644 deploy/k8s/modelrt-secret.yaml create mode 100644 deploy/k8s/modelrt-service.yaml create mode 100644 deploy/k8s/promtail-configmap.yaml create mode 100644 deploy/k8s/promtail-daemonset.yaml create mode 100644 deploy/k8s/promtail-rbac.yaml create mode 100644 deploy/k8s/rabbitmq-config.yaml create mode 100644 deploy/k8s/rabbitmq-deployment.yaml create mode 100644 deploy/k8s/rabbitmq-secret.yaml create mode 100644 deploy/k8s/rabbitmq-service.yaml create mode 100644 deploy/k8s/rabbitmq-users-config.yaml create mode 100644 deploy/k8s/redis-deployment.yaml create mode 100644 deploy/k8s/redis-service.yaml create mode 100644 mq/emit.go create mode 100644 mq/event/task_event_gen.go create mode 100644 mq/publish_message.go diff --git a/constants/event.go b/constants/event.go index 11dbf29..293ca25 100644 --- a/constants/event.go +++ b/constants/event.go @@ -88,3 +88,10 @@ const ( // EventCriticalUpDownLimitCategroy define category for critical up and down limit event EventCriticalUpDownLimitCategroy = "event.critical.updown.limit" ) + +const ( + // EventTaskGeneralTestCategory define category for test task event + EventTaskGeneralTestCategory = "event.general.task.test" + // EventTaskGeneralTopologyAnalyzeCategory define category for topology analyze task event + EventTaskGeneralTopologyAnalyzeCategory = "event.general.task.topology_analyze" +) diff --git a/constants/message.go b/constants/message.go new file mode 100644 index 0000000..7b6fb1f --- /dev/null +++ b/constants/message.go @@ -0,0 +1,33 @@ +// Package constants define constant variable +package constants + +const ( + // MessageExchangeName define exchange name for message + MessageExchangeName = "message-exchange" + // MessageDeadExchangeName define dead letter exchange name for message + MessageDeadExchangeName = "message-dead-letter-exchange" +) + +const ( + // MessageRoutingKey define binding routing key pattern for the message queue (matches all message.* categories) + MessageRoutingKey = "message.#" + // MessageDeadRoutingKey define binding routing key for the message dead letter queue + MessageDeadRoutingKey = "#" + // MessageQueueName define queue name for message + MessageQueueName = "message-queue" + // MessageDeadQueueName define dead letter queue name for message + MessageDeadQueueName = "message-dead-letter-queue" +) + +const ( + // MessageTaskSubmittedCategory define category for task submitted message + MessageTaskSubmittedCategory = "message.task.submitted" + // MessageTaskRunningCategory define category for task running message + MessageTaskRunningCategory = "message.task.running" + // MessageTaskCompletedCategory define category for task completed message + MessageTaskCompletedCategory = "message.task.completed" + // MessageTaskFailedCategory define category for task failed message + MessageTaskFailedCategory = "message.task.failed" + // MessageTaskCancelledCategory define category for task cancelled message + MessageTaskCancelledCategory = "message.task.cancelled" +) diff --git a/deploy/deploy.md b/deploy/deploy.md index 0080b23..8930c91 100644 --- a/deploy/deploy.md +++ b/deploy/deploy.md @@ -45,6 +45,68 @@ docker ps -a ｜grep postgres docker logs postgres ``` +#### 1.4 初始化异步任务表 + +$\text{PostgreSQL}$ 启动后执行以下建表语句，创建异步任务系统所需的两张表： + +```sql +-- ========================================== +-- 表: async_task +-- 说明: 存储异步任务的生命周期跟踪信息 +-- ========================================== +CREATE TABLE IF NOT EXISTS async_task ( + task_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + task_type VARCHAR(50) NOT NULL, + status VARCHAR(20) NOT NULL, + params JSONB, + created_at BIGINT NOT NULL, + finished_at BIGINT, + started_at BIGINT, + execution_time BIGINT, + progress INTEGER, + retry_count INTEGER DEFAULT 0, + max_retry_count INTEGER DEFAULT 3, + next_retry_time BIGINT, + retry_delay INTEGER DEFAULT 5000, + priority INTEGER DEFAULT 5, + queue_name VARCHAR(100) DEFAULT 'default', + worker_id VARCHAR(50), + failure_reason TEXT, + stack_trace TEXT, + created_by VARCHAR(100) +); + +CREATE INDEX IF NOT EXISTS idx_async_task_task_type ON async_task(task_type); +CREATE INDEX IF NOT EXISTS idx_async_task_status ON async_task(status); +CREATE INDEX IF NOT EXISTS idx_async_task_created_at ON async_task(created_at); +CREATE INDEX IF NOT EXISTS idx_async_task_finished_at ON async_task(finished_at); +CREATE INDEX IF NOT EXISTS idx_async_task_started_at ON async_task(started_at); +CREATE INDEX IF NOT EXISTS idx_async_task_next_retry_time ON async_task(next_retry_time); +CREATE INDEX IF NOT EXISTS idx_async_task_priority ON async_task(priority); +CREATE INDEX IF NOT EXISTS idx_async_task_status_retry ON async_task(status, next_retry_time) + WHERE status = 'FAILED' AND next_retry_time IS NOT NULL; + +-- ========================================== +-- 表: async_task_result +-- 说明: 存储异步任务的执行结果 +-- ========================================== +CREATE TABLE IF NOT EXISTS async_task_result ( + task_id UUID PRIMARY KEY, + result JSONB, + error_code INTEGER, + error_message TEXT, + error_detail JSONB, + execution_time BIGINT NOT NULL DEFAULT 0, + memory_usage BIGINT, + cpu_usage DOUBLE PRECISION, + retry_count INTEGER DEFAULT 0, + completed_at BIGINT NOT NULL +); + +COMMENT ON TABLE async_task IS '异步任务生命周期跟踪表'; +COMMENT ON TABLE async_task_result IS '异步任务执行结果表'; +``` + ### 2\. 部署 Redis Stack Server 我们将使用 `redis/redis-stack-server:latest` 镜像该镜像内置了 $\text{Redisearch}$ 模块，用于 $\text{ModelRT}$ 项目中补全功能 @@ -401,15 +463,453 @@ go build -o model-rt main.go 在发现控制台输出如下信息`starting ModelRT server` 后即代表服务启动成功 -### 4\. 后续操作（停止与清理） +### 4\. 部署基础依赖（Kubernetes） -#### 4.1 停止容器 +Redis 和 RabbitMQ 部署在 Minikube 中，YAML 文件位于 `deploy/k8s/`。RabbitMQ 启用双向 TLS（mTLS），客户端以 X.509 证书的 CN 字段作为用户名进行认证。 + +#### 4.1 部署 Redis + +```bash +kubectl apply -f deploy/k8s/redis-deployment.yaml +kubectl apply -f deploy/k8s/redis-service.yaml +``` + +| 参数 | 值 | 说明 | +| :--- | :--- | :--- | +| **镜像** | `redis/redis-stack-server:latest` | 内置 Redisearch 模块 | +| **NodePort** | `30001` | 集群外访问端口 | + +#### 4.2 RabbitMQ TLS 证书生成 + +RabbitMQ 配置为仅允许 TLS 连接（`listeners.tcp = none`），所有客户端须持有由同一 CA 签发的证书。 + +##### 4.2.1 生成根 CA + +```bash +# 克隆 tls-gen 工具 +git clone https://github.com/rabbitmq/tls-gen.git +cd tls-gen/basic + +# 生成根 CA（结果在 result/ 目录） +make CN=rabbitmq-server +# ca_certificate.pem 和 ca_key.pem 生成于 result/ +``` + +##### 4.2.2 生成服务器证书 + +服务器证书需包含 SAN（Subject Alternative Name），使其同时匹配集群内 DNS 和 Minikube IP。 + +创建 `server.cnf`： + +```text +[req] +distinguished_name = req_distinguished_name +prompt = no + +[req_distinguished_name] +C = CN +ST = Beijing +L = Beijing +O = coslight +CN = rabbitmq-server + +[v3_server] +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth, clientAuth +subjectAltName = @alt_names + +[alt_names] +DNS.1 = rabbitmq-server +DNS.2 = rabbitmq-service.default.svc.cluster.local +DNS.3 = localhost +IP.1 = 192.168.49.2 +IP.2 = 127.0.0.1 +``` + +生成证书： + +```bash +# 将 ca_certificate.pem 和 ca_key.pem（即 cakey.pem）放在当前目录 +openssl genrsa -out server_key.pem 2048 + +openssl req -new -key server_key.pem -out server_cert.csr -config server.cnf + +openssl x509 -req -in server_cert.csr \ + -CA ca_certificate.pem -CAkey cakey.pem -CAcreateserial \ + -out server_certificate.pem -days 730 -sha256 \ + -extfile server.cnf -extensions v3_server + +rm server_cert.csr +``` + +##### 4.2.3 生成 ModelRT 客户端证书 + +CN 必须与 RabbitMQ 中注册的用户名一致（`modelrt-client`）。 + +创建 `modelrt.cnf`： + +```text +[req] +distinguished_name = req_distinguished_name +prompt = no + +[req_distinguished_name] +C = CN +ST = Beijing +L = Beijing +O = coslight +CN = modelrt-client + +[v3_client] +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth +``` + +生成证书： + +```bash +openssl genrsa -out modelrt_client_key.pem 2048 + +openssl req -new -key modelrt_client_key.pem \ + -out modelrt_client.csr -config modelrt.cnf + +openssl x509 -req -in modelrt_client.csr \ + -CA ca_certificate.pem -CAkey cakey.pem -CAcreateserial \ + -out modelrt_client_cert.pem -days 365 \ + -extensions v3_client -extfile modelrt.cnf + +rm modelrt_client.csr +``` + +##### 4.2.4 生成 EventRT 客户端证书 + +创建 `eventrt.cnf`（CN 改为 `eventrt-client`）： + +```text +[req] +distinguished_name = req_distinguished_name +prompt = no + +[req_distinguished_name] +C = CN +ST = Beijing +L = Beijing +O = coslight +CN = eventrt-client + +[v3_client] +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth +``` + +生成证书： + +```bash +openssl genrsa -out eventrt_client_key.pem 2048 + +openssl req -new -key eventrt_client_key.pem \ + -out eventrt_client.csr -config eventrt.cnf + +openssl x509 -req -in eventrt_client.csr \ + -CA ca_certificate.pem -CAkey cakey.pem -CAcreateserial \ + -out eventrt_client_cert.pem -days 365 \ + -extensions v3_client -extfile eventrt.cnf + +rm eventrt_client.csr +``` + +##### 4.2.5 验证证书 + +```bash +# 验证服务器证书 +openssl verify -CAfile ca_certificate.pem server_certificate.pem + +# 验证客户端证书 +openssl verify -CAfile ca_certificate.pem modelrt_client_cert.pem +openssl verify -CAfile ca_certificate.pem eventrt_client_cert.pem + +# 查看证书详情（确认 CN 和 SAN） +openssl x509 -in server_certificate.pem -noout -subject -ext subjectAltName +openssl x509 -in modelrt_client_cert.pem -noout -subject +openssl x509 -in eventrt_client_cert.pem -noout -subject +``` + +#### 4.3 部署 RabbitMQ + +##### 4.3.1 创建证书 Secret + +将服务器端三个证书文件打包为 K8s Secret（在证书文件所在目录执行）： + +```bash +kubectl create secret generic rabbitmq-certs \ + --from-file=ca_certificate.pem=./ca_certificate.pem \ + --from-file=server_certificate.pem=./server_certificate.pem \ + --from-file=server_key.pem=./server_key.pem +``` + +##### 4.3.2 部署 + +```bash +kubectl apply -f deploy/k8s/rabbitmq-secret.yaml +kubectl apply -f deploy/k8s/rabbitmq-config.yaml +kubectl apply -f deploy/k8s/rabbitmq-users-config.yaml +kubectl apply -f deploy/k8s/rabbitmq-deployment.yaml +kubectl apply -f deploy/k8s/rabbitmq-service.yaml +``` + +##### 4.3.3 端口汇总 + +| 端口 | NodePort | 说明 | +| :--- | :--- | :--- | +| `5671` | `30671` | AMQP over TLS（客户端连接） | +| `5672` | `30672` | AMQP 明文（内部备用，生产禁用） | +| `15671` | `31671` | Management UI over TLS | +| `15672` | `31672` | Management UI 明文（内部备用） | + +##### 4.3.4 用户与权限说明 + +用户定义在 `rabbitmq-users-config.yaml` 的 `definitions.json` 中，通过 `load_definitions` 启动时自动加载： + +| 用户 | 认证方式 | 权限 | 说明 | +| :--- | :--- | :--- | :--- | +| `coslight` | 密码 | administrator | 管理员，密码在 rabbitmq-secret.yaml | +| `modelrt-client` | X.509 证书（CN） | configure/read/write | ModelRT 服务专用 | +| `eventrt-client` | X.509 证书（CN） | configure/read/write | EventRT 服务专用 | +| `web-client` | X.509 证书（CN） | read/write | Web 客户端 | + +> **注意：** 证书认证用户的 `password_hash` 留空；RabbitMQ 通过 `ssl_cert_login_from = common_name` 将证书 CN 映射为用户名。 + +### 5\. 部署 ModelRT（Kubernetes） + +所有资源部署在 `default` 命名空间，YAML 文件位于 `deploy/k8s/`。 + +#### 5.1 构建并推送镜像 + +```bash +# 在项目根目录执行 +docker build -f deploy/dockerfile/modelrt.Dockerfile -t coslight/modelrt:latest . + +# 推送到镜像仓库（或直接加载到 Minikube） +minikube image load coslight/modelrt:latest +``` + +#### 5.2 创建客户端证书 Secret + +在 RabbitMQ TLS 证书生成完成后（见 4.2），进入证书文件所在目录执行： + +```bash +sh deploy/k8s/modelrt-certs-secret.sh +``` + +该脚本等价于： + +```bash +kubectl create secret generic modelrt-certs \ + --from-file=ca_certificate.pem=./ca_certificate.pem \ + --from-file=modelrt_client_cert.pem=./modelrt_client_cert.pem \ + --from-file=modelrt_client_key.pem=./modelrt_client_key.pem +``` + +#### 5.3 部署 + +```bash +kubectl apply -f deploy/k8s/modelrt-secret.yaml +kubectl apply -f deploy/k8s/modelrt-configmap.yaml +kubectl apply -f deploy/k8s/modelrt-deployment.yaml +kubectl apply -f deploy/k8s/modelrt-service.yaml +``` + +#### 5.4 配置说明 + +| 配置项 | 方式 | 说明 | +| :--- | :--- | :--- | +| `postgres.password` | Secret `modelrt-secret` | 不写入 ConfigMap | +| `service.secret_key` | Secret `modelrt-secret` | 不写入 ConfigMap | +| RabbitMQ 客户端证书 | Secret `modelrt-certs` | 挂载至 `/app/configs/certs/` | +| `config.yaml` 其余配置 | ConfigMap `modelrt-config` | 所有 host 已替换为 K8s service 名 | +| `K8S_NAMESPACE` / `K8S_NODE_NAME` | Downward API | 注入至日志全局字段 | + +> **注意：** `modelrt-configmap.yaml` 中 `postgres.password` 和 `service.secret_key` 留空，实际值由容器启动时的环境变量 `POSTGRES_PASSWORD` / `SERVICE_SECRET_KEY` 注入，应用需读取这两个环境变量覆盖 config 中的空值。若应用当前仅读取文件配置，可直接将值填入 `modelrt-secret.yaml` 并在 ConfigMap 中引用，或在 ConfigMap 中直接填写。 + +#### 5.5 状态检查 + +```bash +# 查看 Pod 状态 +kubectl get pods -l app=modelrt + +# 查看启动日志 +kubectl logs -l app=modelrt --tail=50 + +# 查看 Service +kubectl get svc modelrt-service +``` + +#### 5.6 端口汇总 + +| NodePort | 说明 | +| :--- | :--- | +| `30080` | ModelRT HTTP API，SSH 隧道本地端口 `8080` | + +#### 5.7 清理 + +```bash +kubectl delete -f deploy/k8s/modelrt-service.yaml \ + -f deploy/k8s/modelrt-deployment.yaml \ + -f deploy/k8s/modelrt-configmap.yaml \ + -f deploy/k8s/modelrt-secret.yaml +kubectl delete secret modelrt-certs +``` + +### 6\. 部署可观测性栈（Kubernetes） + +在 $\text{Kubernetes}$ 集群中部署 $\text{Jaeger}$（链路追踪）+ $\text{Loki + Promtail + Grafana}$（日志可视化）。所有资源部署在 `default` 命名空间，$\text{YAML}$ 文件位于 `deploy/k8s/`。 + +#### 6.1 部署 Jaeger + +```bash +kubectl apply -f deploy/k8s/jaeger-deployment.yaml +kubectl apply -f deploy/k8s/jaeger-service.yaml +``` + +#### 6.2 部署 Loki + +```bash +kubectl apply -f deploy/k8s/loki-configmap.yaml +kubectl apply -f deploy/k8s/loki-pvc.yaml +kubectl apply -f deploy/k8s/loki-deployment.yaml +kubectl apply -f deploy/k8s/loki-service.yaml +``` + +#### 6.3 部署 Promtail + +```bash +kubectl apply -f deploy/k8s/promtail-rbac.yaml +kubectl apply -f deploy/k8s/promtail-configmap.yaml +kubectl apply -f deploy/k8s/promtail-daemonset.yaml +``` + +#### 6.4 部署 Grafana + +```bash +kubectl apply -f deploy/k8s/grafana-configmap.yaml +kubectl apply -f deploy/k8s/grafana-deployment.yaml +kubectl apply -f deploy/k8s/grafana-service.yaml +``` + +#### 6.5 一键部署 + +```bash +kubectl apply -f deploy/k8s/jaeger-deployment.yaml \ + -f deploy/k8s/jaeger-service.yaml \ + -f deploy/k8s/loki-configmap.yaml \ + -f deploy/k8s/loki-pvc.yaml \ + -f deploy/k8s/loki-deployment.yaml \ + -f deploy/k8s/loki-service.yaml \ + -f deploy/k8s/promtail-rbac.yaml \ + -f deploy/k8s/promtail-configmap.yaml \ + -f deploy/k8s/promtail-daemonset.yaml \ + -f deploy/k8s/grafana-configmap.yaml \ + -f deploy/k8s/grafana-deployment.yaml \ + -f deploy/k8s/grafana-service.yaml +``` + +#### 6.6 状态检查 + +```bash +# 查看所有 Pod 状态 +kubectl get pods + +# 查看所有 Service 及 NodePort +kubectl get svc +``` + +#### 6.7 端口汇总 + +| 服务 | NodePort | 访问地址 | 说明 | +| :--- | :--- | :--- | :--- | +| **Jaeger UI** | `31686` | `http://:31686` | 链路追踪查询界面 | +| **Loki** | `31100` | `http://:31100` | 日志 HTTP API | +| **Grafana** | `31000` | `http://:31000` | 可视化界面，账号 `admin / coslight` | +| **OTLP gRPC** | `31317` | `:31317` | ModelRT OTel 上报地址（gRPC） | +| **OTLP HTTP** | `31318` | `http://:31318` | ModelRT OTel 上报地址（HTTP） | + +#### 6.8 清理 + +```bash +kubectl delete -f deploy/k8s/ +``` + +### 7\. Mac 本地访问（SSH 隧道） + +$\text{ModelRT / EventRT}$ 在 $\text{Mac}$ 本地运行时，依赖的 $\text{RabbitMQ}$、$\text{Redis}$、$\text{Jaeger}$、$\text{Loki}$、$\text{Grafana}$ 均部署在 $\text{Ubuntu}$ 宿主机（`192.168.1.101`）上的 $\text{Minikube}$（`192.168.49.2`）中。由于 $\text{Minikube}$ 网络不直接对外暴露，需通过 $\text{SSH}$ 本地端口转发建立访问隧道。 + +#### 7.1 网络拓扑 + +``` text +Mac 本地端口 ──SSH隧道──▶ Ubuntu 宿主机 (192.168.1.101) ──▶ Minikube NodePort (192.168.49.2) +``` + +#### 7.2 建立隧道 + +```bash +ssh -L 5671:192.168.49.2:30671 \ + -L 15671:192.168.49.2:31671 \ + -L 6379:192.168.49.2:30001 \ + -L 4318:192.168.49.2:31318 \ + -L 16686:192.168.49.2:31686 \ + -L 3100:192.168.49.2:31100 \ + -L 3000:192.168.49.2:31000 \ + douxu@192.168.1.101 +``` + +如需后台静默运行（不占用终端）： + +```bash +ssh -fN \ + -L 5671:192.168.49.2:30671 \ + -L 15671:192.168.49.2:31671 \ + -L 6379:192.168.49.2:30001 \ + -L 4318:192.168.49.2:31318 \ + -L 16686:192.168.49.2:31686 \ + -L 3100:192.168.49.2:31100 \ + -L 3000:192.168.49.2:31000 \ + douxu@192.168.1.101 +``` + +#### 7.3 端口映射说明 + +| Mac 本地端口 | Minikube NodePort | 服务 | 说明 | +| :--- | :--- | :--- | :--- | +| `5671` | `30671` | RabbitMQ AMQP | ModelRT / EventRT 消息队列连接 | +| `15671` | `31671` | RabbitMQ Management | RabbitMQ 管理界面 `http://localhost:15671` | +| `6379` | `30001` | Redis | 分布式锁 / 数据存储 | +| `4318` | `31318` | OTLP HTTP | OTel Trace 上报（Jaeger Collector） | +| `16686` | `31686` | Jaeger UI | 链路追踪查询 `http://localhost:16686` | +| `3100` | `31100` | Loki | 日志查询 API | +| `3000` | `31000` | Grafana | 可视化界面 `http://localhost:3000` | + +> **注意：** 隧道建立后，本地配置文件中所有服务地址均填 `localhost:<本地端口>`，无需修改即可在 $\text{Mac}$ 上直接运行服务。 + +#### 7.4 关闭隧道 + +前台运行时直接 `Ctrl+C`；后台运行时查找并终止进程： + +```bash +# 找到 ssh 隧道进程 +ps aux | grep "ssh -fN" +# 终止（替换为实际 PID） +kill +``` + +### 8\. 后续操作（停止与清理） + +#### 8.1 停止容器 ```bash docker stop postgres redis ``` -#### 4.2 删除容器（删除后数据将丢失） +#### 8.2 删除容器（删除后数据将丢失） ```bash docker rm postgres redis diff --git a/deploy/dockerfile/modelrt.Dockerfile b/deploy/dockerfile/modelrt.Dockerfile index e251642..c091aae 100644 --- a/deploy/dockerfile/modelrt.Dockerfile +++ b/deploy/dockerfile/modelrt.Dockerfile @@ -1,19 +1,35 @@ FROM golang:1.24-alpine AS builder +RUN apk --no-cache upgrade WORKDIR /app -COPY go.mod . -COPY go.sum . +COPY go.mod go.sum ./ RUN GOPROXY="https://goproxy.cn,direct" go mod download COPY . . -RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o modelrt main.go +RUN CGO_ENABLED=0 GOOS=linux go build \ + -ldflags="-s -w" \ + -trimpath \ + -mod=readonly \ + -o modelrt main.go -FROM alpine:latest -WORKDIR /app +# Prepare runtime dependencies in a pinned Alpine stage so they can be +# copied into scratch without pulling any vulnerable OS packages at run time. +FROM alpine:3.21 AS certs ARG USER_ID=1000 -RUN adduser -D -u ${USER_ID} modelrt +RUN apk --no-cache add ca-certificates tzdata && \ + adduser -D -u ${USER_ID} modelrt + +FROM scratch +# CA certificates required for TLS connections (RabbitMQ amqps://) +COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +# Timezone data +COPY --from=certs /usr/share/zoneinfo /usr/share/zoneinfo +# Non-root user/group definitions +COPY --from=certs /etc/passwd /etc/passwd +COPY --from=certs /etc/group /etc/group + +WORKDIR /app COPY --from=builder /app/modelrt ./modelrt COPY configs/config.example.yaml ./configs/config.example.yaml -RUN chown -R modelrt:modelrt /app -RUN chmod +x /app/modelrt + USER modelrt -CMD ["/app/modelrt", "-modelRT_config_dir=/app/configs"] \ No newline at end of file +CMD ["/app/modelrt", "-modelRT_config_dir=/app/configs"] diff --git a/deploy/k8s/grafana-configmap.yaml b/deploy/k8s/grafana-configmap.yaml new file mode 100644 index 0000000..76b2cb0 --- /dev/null +++ b/deploy/k8s/grafana-configmap.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + namespace: default +data: + datasources.yaml: | + apiVersion: 1 + datasources: + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: true + jsonData: + # derivedFields: 从日志的 traceID 字段生成跳转链接到 Jaeger + derivedFields: + - matcherRegex: '"traceID":\s*"([a-f0-9]+)"' + name: TraceID + url: http://127.0.0.1:16686/trace/$${__value.raw} + targetBlank: true + - name: Jaeger + type: jaeger + uid: jaeger + access: proxy + url: http://jaeger:16686 diff --git a/deploy/k8s/grafana-deployment.yaml b/deploy/k8s/grafana-deployment.yaml new file mode 100644 index 0000000..9f23045 --- /dev/null +++ b/deploy/k8s/grafana-deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + containers: + - name: grafana + image: grafana/grafana:10.4.2 + ports: + - containerPort: 3000 + env: + - name: GF_SECURITY_ADMIN_USER + value: "coslight" + - name: GF_SECURITY_ADMIN_PASSWORD + value: "coslight@tj" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "false" + volumeMounts: + - name: datasources + mountPath: /etc/grafana/provisioning/datasources + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + volumes: + - name: datasources + configMap: + name: grafana-datasources diff --git a/deploy/k8s/grafana-service.yaml b/deploy/k8s/grafana-service.yaml new file mode 100644 index 0000000..1cc3782 --- /dev/null +++ b/deploy/k8s/grafana-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: default +spec: + ports: + - name: http + port: 3000 + targetPort: 3000 + nodePort: 31000 # Grafana UI: http://:31000 + selector: + app: grafana + type: NodePort diff --git a/deploy/jaeger.yaml b/deploy/k8s/jaeger-deployment.yaml similarity index 52% rename from deploy/jaeger.yaml rename to deploy/k8s/jaeger-deployment.yaml index 8dac477..c0444b7 100644 --- a/deploy/jaeger.yaml +++ b/deploy/k8s/jaeger-deployment.yaml @@ -1,31 +1,3 @@ -apiVersion: v1 -kind: Service -metadata: - name: jaeger - labels: - app: jaeger -spec: - ports: - - name: ui - port: 16686 - targetPort: 16686 - nodePort: 31686 # Jaeger UI，浏览器访问 http://:31686 - - name: collector-http - port: 14268 - targetPort: 14268 - nodePort: 31268 # Jaeger 原生 HTTP collector（非 OTel） - - name: otlp-http - port: 4318 - targetPort: 4318 - nodePort: 31318 # OTLP HTTP，集群外使用 :31318 - - name: otlp-grpc - port: 4317 - targetPort: 4317 - nodePort: 31317 # OTLP gRPC，集群外使用 :31317 - selector: - app: jaeger - type: NodePort ---- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deploy/k8s/jaeger-service.yaml b/deploy/k8s/jaeger-service.yaml new file mode 100644 index 0000000..d1e4779 --- /dev/null +++ b/deploy/k8s/jaeger-service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + name: jaeger + labels: + app: jaeger +spec: + ports: + - name: ui + port: 16686 + targetPort: 16686 + nodePort: 31686 # Jaeger UI，浏览器访问 http://:31686 + - name: collector-http + port: 14268 + targetPort: 14268 + nodePort: 31268 # Jaeger 原生 HTTP collector（非 OTel） + - name: otlp-http + port: 4318 + targetPort: 4318 + nodePort: 31318 # OTLP HTTP，集群外使用 :31318 + - name: otlp-grpc + port: 4317 + targetPort: 4317 + nodePort: 31317 # OTLP gRPC，集群外使用 :31317 + selector: + app: jaeger + type: NodePort diff --git a/deploy/k8s/loki-configmap.yaml b/deploy/k8s/loki-configmap.yaml new file mode 100644 index 0000000..b126a9f --- /dev/null +++ b/deploy/k8s/loki-configmap.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-config + namespace: default +data: + loki.yaml: | + auth_enabled: false + + server: + http_listen_port: 3100 + + ingester: + wal: + enabled: true + dir: /loki/wal # 指向 PVC 挂载路径，避免在容器根目录创建 /wal 时 permission denied + lifecycler: + ring: + kvstore: + store: inmemory + replication_factor: 1 + chunk_idle_period: 5m + chunk_retain_period: 30s + + schema_config: + configs: + - from: 2024-01-01 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + + storage_config: + boltdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/cache + shared_store: filesystem + filesystem: + directory: /loki/chunks + + limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + + compactor: + working_directory: /loki/compactor + shared_store: filesystem diff --git a/deploy/k8s/loki-deployment.yaml b/deploy/k8s/loki-deployment.yaml new file mode 100644 index 0000000..63ff925 --- /dev/null +++ b/deploy/k8s/loki-deployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: loki + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: loki + template: + metadata: + labels: + app: loki + spec: + securityContext: + fsGroup: 10001 # 使 PVC 挂载目录对 Loki 默认用户（UID 10001）可写 + runAsUser: 10001 + runAsGroup: 10001 + containers: + - name: loki + image: grafana/loki:2.9.4 + args: + - -config.file=/etc/loki/loki.yaml + ports: + - containerPort: 3100 + volumeMounts: + - name: config + mountPath: /etc/loki + - name: storage + mountPath: /loki + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + volumes: + - name: config + configMap: + name: loki-config + - name: storage + persistentVolumeClaim: + claimName: loki-pvc diff --git a/deploy/k8s/loki-pvc.yaml b/deploy/k8s/loki-pvc.yaml new file mode 100644 index 0000000..f329a08 --- /dev/null +++ b/deploy/k8s/loki-pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: loki-pvc + namespace: default +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/deploy/k8s/loki-service.yaml b/deploy/k8s/loki-service.yaml new file mode 100644 index 0000000..e0df759 --- /dev/null +++ b/deploy/k8s/loki-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: loki + namespace: default +spec: + ports: + - name: http + port: 3100 + targetPort: 3100 + nodePort: 31100 # 集群外访问: http://:31100 + selector: + app: loki + type: NodePort diff --git a/deploy/k8s/modelrt-certs-secret.sh b/deploy/k8s/modelrt-certs-secret.sh new file mode 100644 index 0000000..33a1bbd --- /dev/null +++ b/deploy/k8s/modelrt-certs-secret.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Create the modelrt client certificate secret. +# Run this script from the directory that contains the three cert files, +# or adjust the paths below to point at the actual files. +# +# Expected files (generated during RabbitMQ TLS setup): +# ca_certificate.pem +# modelrt_client_cert.pem +# modelrt_client_key.pem + +kubectl create secret generic modelrt-certs \ + --from-file=ca_certificate.pem=./ca_certificate.pem \ + --from-file=modelrt_client_cert.pem=./modelrt_client_cert.pem \ + --from-file=modelrt_client_key.pem=./modelrt_client_key.pem diff --git a/deploy/k8s/modelrt-configmap.yaml b/deploy/k8s/modelrt-configmap.yaml new file mode 100644 index 0000000..b1a6afc --- /dev/null +++ b/deploy/k8s/modelrt-configmap.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: modelrt-config +data: + config.yaml: | + postgres: + host: "192.168.1.101" + port: 5432 + database: "demo" + user: "postgres" + password: "" # injected via env POSTGRES_PASSWORD + + rabbitmq: + ca_cert_path: "/app/configs/certs/ca_certificate.pem" + client_key_path: "/app/configs/certs/modelrt_client_key.pem" + client_key_password: "" + client_cert_path: "/app/configs/certs/modelrt_client_cert.pem" + insecure_skip_verify: false + server_name: "rabbitmq-server" + user: "" + password: "" + host: "rabbitmq-service" + port: 5671 + + logger: + mode: "production" + level: "info" + filepath: "" + maxsize: 100 + maxbackups: 5 + maxage: 30 + compress: false + loki: + endpoint: "" # Promtail handles log collection in K8s, direct push disabled + + otel: + endpoint: "jaeger:4318" + insecure: true + + ants: + parse_concurrent_quantity: 10 + rtd_receive_concurrent_quantity: 10 + + async_task: + worker_pool_size: 10 + queue_consumer_count: 2 + max_retry_count: 3 + retry_initial_delay: 1s + retry_max_delay: 5m + health_check_interval: 30s + + locker_redis: + addr: "redis-service:6379" + password: "" + db: 1 + poolsize: 50 + dial_timeout: 10 + read_timeout: 10 + write_timeout: 10 + + storage_redis: + addr: "redis-service:6379" + password: "" + db: 0 + poolsize: 50 + dial_timeout: 10 + read_timeout: 10 + write_timeout: 10 + + base: + grid_id: 1 + zone_id: 1 + station_id: 1 + + service: + service_addr: ":8080" + service_name: "modelRT" + secret_key: "" # injected via env SERVICE_SECRET_KEY + deploy_env: "production" + + dataRT: + host: "http://127.0.0.1" + port: 8888 + polling_api: "datart/getPointData" + polling_api_method: "GET" diff --git a/deploy/k8s/modelrt-deployment.yaml b/deploy/k8s/modelrt-deployment.yaml new file mode 100644 index 0000000..38a9a23 --- /dev/null +++ b/deploy/k8s/modelrt-deployment.yaml @@ -0,0 +1,90 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: modelrt + labels: + app: modelrt +spec: + replicas: 1 + selector: + matchLabels: + app: modelrt + template: + metadata: + labels: + app: modelrt + spec: + containers: + - name: modelrt + image: coslight/modelrt:latest + imagePullPolicy: IfNotPresent + args: + - "-modelRT_config_dir=/app/configs" + - "-modelRT_config_name=config" + - "-modelRT_config_type=yaml" + ports: + - containerPort: 8080 + env: + # Downward API — injected into every log line by logger/zap.go containerFields() + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # HOSTNAME is set automatically by K8s to the pod name + # Sensitive values injected from Secret so they stay out of ConfigMap + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: modelrt-secret + key: postgres-password + - name: SERVICE_SECRET_KEY + valueFrom: + secretKeyRef: + name: modelrt-secret + key: secret-key + volumeMounts: + - name: config + mountPath: /app/configs/config.yaml + subPath: config.yaml + readOnly: true + - name: certs + mountPath: /app/configs/certs + readOnly: true + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + runAsUser: 1000 + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + livenessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + volumes: + - name: config + configMap: + name: modelrt-config + - name: certs + secret: + secretName: modelrt-certs diff --git a/deploy/k8s/modelrt-secret.yaml b/deploy/k8s/modelrt-secret.yaml new file mode 100644 index 0000000..e078e4d --- /dev/null +++ b/deploy/k8s/modelrt-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: modelrt-secret +type: Opaque +stringData: + postgres-password: "coslight" + secret-key: "modelrt_key" diff --git a/deploy/k8s/modelrt-service.yaml b/deploy/k8s/modelrt-service.yaml new file mode 100644 index 0000000..88b2cf7 --- /dev/null +++ b/deploy/k8s/modelrt-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: modelrt-service + labels: + app: modelrt +spec: + type: NodePort + selector: + app: modelrt + ports: + - name: http + port: 8080 + targetPort: 8080 + nodePort: 30080 diff --git a/deploy/k8s/promtail-configmap.yaml b/deploy/k8s/promtail-configmap.yaml new file mode 100644 index 0000000..0ccf089 --- /dev/null +++ b/deploy/k8s/promtail-configmap.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: promtail-config + namespace: default +data: + promtail.yaml: | + server: + http_listen_port: 9080 + grpc_listen_port: 0 + + positions: + filename: /tmp/positions.yaml + + clients: + - url: http://loki:3100/loki/api/v1/push + + scrape_configs: + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + pipeline_stages: + # 解析 zap 输出的 JSON 日志，提取结构化字段 + - json: + expressions: + level: level + traceID: traceID + spanID: spanID + caller: caller + pod: pod + namespace: namespace + node: node + # 将关键字段提升为 Loki Label,支持在 Grafana 中按实例/Trace 过滤 + - labels: + level: + traceID: + pod: + namespace: + node: + relabel_configs: + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_container_name] + target_label: container + - source_labels: [__meta_kubernetes_pod_label_app] + target_label: app + # 只采集有 app label 的 Pod + - source_labels: [__meta_kubernetes_pod_label_app] + action: keep + regex: .+ diff --git a/deploy/k8s/promtail-daemonset.yaml b/deploy/k8s/promtail-daemonset.yaml new file mode 100644 index 0000000..eedf72d --- /dev/null +++ b/deploy/k8s/promtail-daemonset.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: promtail + namespace: default +spec: + selector: + matchLabels: + app: promtail + template: + metadata: + labels: + app: promtail + spec: + serviceAccountName: promtail + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: promtail + image: grafana/promtail:2.9.4 + args: + - -config.file=/etc/promtail/promtail.yaml + ports: + - containerPort: 9080 + volumeMounts: + - name: config + mountPath: /etc/promtail + - name: varlog + mountPath: /var/log + readOnly: true + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 50m + memory: 64Mi + volumes: + - name: config + configMap: + name: promtail-config + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers diff --git a/deploy/k8s/promtail-rbac.yaml b/deploy/k8s/promtail-rbac.yaml new file mode 100644 index 0000000..2433f23 --- /dev/null +++ b/deploy/k8s/promtail-rbac.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: promtail + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: promtail +rules: + - apiGroups: [""] + resources: ["nodes", "nodes/proxy", "services", "endpoints", "pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: promtail +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: promtail +subjects: + - kind: ServiceAccount + name: promtail + namespace: default diff --git a/deploy/k8s/rabbitmq-config.yaml b/deploy/k8s/rabbitmq-config.yaml new file mode 100644 index 0000000..a5cbad7 --- /dev/null +++ b/deploy/k8s/rabbitmq-config.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-config +data: + rabbitmq.conf: | + # 确保允许PLAIN认证 + auth_mechanisms.1 = PLAIN + auth_mechanisms.2 = AMQPLAIN + auth_mechanisms.3 = EXTERNAL + # 允许admin用户通过远程方式连接 + loopback_users.admin = false + # 默认心跳和监听配置可在此扩展 + # 确定 ssl 连接时验证使用的用户名 + ssl_cert_login_from = common_name + # 开启此项配置会导致只能通过TLS端口访问 + listeners.tcp = none + listeners.ssl.default = 5671 + # default user config + load_definitions = /etc/rabbitmq/definitions.json + # ssl config + ssl_options.cacertfile = /etc/rabbitmq/certs/ca_certificate.pem + ssl_options.certfile = /etc/rabbitmq/certs/server_certificate.pem + ssl_options.keyfile = /etc/rabbitmq/certs/server_key.pem + ssl_options.verify = verify_peer + ssl_options.fail_if_no_peer_cert = true + # management config + management.ssl.port = 15671 + management.ssl.cacertfile = /etc/rabbitmq/certs/ca_certificate.pem + management.ssl.certfile = /etc/rabbitmq/certs/server_certificate.pem + management.ssl.keyfile = /etc/rabbitmq/certs/server_key.pem + management.ssl.verify = verify_peer + management.ssl.fail_if_no_peer_cert = true diff --git a/deploy/k8s/rabbitmq-deployment.yaml b/deploy/k8s/rabbitmq-deployment.yaml new file mode 100644 index 0000000..758daca --- /dev/null +++ b/deploy/k8s/rabbitmq-deployment.yaml @@ -0,0 +1,81 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: eventrt-rabbitmq +spec: + replicas: 1 + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + containers: + - name: rabbitmq + image: rabbitmq:4.1.1-management-alpine + ports: + - containerPort: 4369 + - containerPort: 5671 + - containerPort: 5672 # AMQP + - containerPort: 15671 + - containerPort: 15672 # Management UI + - containerPort: 15691 + - containerPort: 15692 + - containerPort: 25672 + env: + - name: RABBITMQ_DEFAULT_USER + valueFrom: + secretKeyRef: + name: rabbitmq-secret + key: rabbitmq-user + - name: RABBITMQ_DEFAULT_PASS + valueFrom: + secretKeyRef: + name: rabbitmq-secret + key: rabbitmq-pass + - name: RABBITMQ_ERLANG_COOKIE + valueFrom: + secretKeyRef: + name: rabbitmq-secret + key: erlang-cookie + - name: RABBITMQ_DEFAULT_VHOST + value: "/" + volumeMounts: + - name: rabbitmq-certs-volume + mountPath: /etc/rabbitmq/certs + readOnly: true + - name: rabbitmq-config-volume + mountPath: /etc/rabbitmq/rabbitmq.conf + subPath: rabbitmq.conf + - name: rabbitmq-config-volume + mountPath: /etc/rabbitmq/advanced.config + subPath: advanced.config + readOnly: true + - name: plugins-config-volume + mountPath: /etc/rabbitmq/enabled_plugins + subPath: enabled_plugins + - name: users-config-volume + mountPath: /etc/rabbitmq/definitions.json + subPath: definitions.json + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + volumes: + - name: rabbitmq-certs-volume + secret: + secretName: rabbitmq-certs + - name: rabbitmq-config-volume + configMap: + name: rabbitmq-config + - name: rabbitmq-advanced-config-volume + configMap: + name: rabbitmq-config + - name: plugins-config-volume + configMap: + name: rabbit-plugins-conf + - name: users-config-volume + configMap: + name: rabbitmq-users-definitions + - name: rabbitmq-data + emptyDir: {} diff --git a/deploy/k8s/rabbitmq-secret.yaml b/deploy/k8s/rabbitmq-secret.yaml new file mode 100644 index 0000000..eae46a1 --- /dev/null +++ b/deploy/k8s/rabbitmq-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: rabbitmq-secret +type: Opaque +stringData: + rabbitmq-user: "coslight" + rabbitmq-pass: "coslight@tj" + erlang-cookie: "secret-erlang-cookie" diff --git a/deploy/k8s/rabbitmq-service.yaml b/deploy/k8s/rabbitmq-service.yaml new file mode 100644 index 0000000..6cdb259 --- /dev/null +++ b/deploy/k8s/rabbitmq-service.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq-service +spec: + type: NodePort # 在 Minikube 中使用 NodePort 方便外部访问 + selector: + app: rabbitmq + ports: + - name: amqp-ssl + protocol: TCP + port: 5671 + targetPort: 5671 + nodePort: 30671 + - name: amqp + protocol: TCP + port: 5672 + targetPort: 5672 + nodePort: 30672 + - name: management-ssl + protocol: TCP + port: 15671 + targetPort: 15671 + nodePort: 31671 + - name: management + protocol: TCP + port: 15672 + targetPort: 15672 + nodePort: 31672 diff --git a/deploy/k8s/rabbitmq-users-config.yaml b/deploy/k8s/rabbitmq-users-config.yaml new file mode 100644 index 0000000..8de5f30 --- /dev/null +++ b/deploy/k8s/rabbitmq-users-config.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-users-definitions +data: + definitions.json: | + { + "users": [ + { + "name": "coslight", + "password_hash": "Gl2XVEJwPwDZQF8ZhsYnvm83wMkdftY3/raxyntdZueyx/Uv", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": ["administrator"] + }, + { + "name": "web-client", + "password_hash": "", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": ["management"] + }, + { + "name": "modelrt-client", + "password_hash": "", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": ["management"] + }, + { + "name": "eventrt-client", + "password_hash": "", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": ["management"] + } + ], + "vhosts": [ { "name": "/" } ], + "permissions": [ + { + "user": "coslight", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "web-client", + "vhost": "/", + "configure": "^$", + "write": ".*", + "read": ".*" + }, + { + "user": "modelrt-client", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "eventrt-client", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + } + ], + "topic_permissions": [], + "parameters": [], + "global_parameters": [ + { + "name": "cluster_name", + "value": "evnetrt-rabbitmq-cluster" + } + ], + "policies": [], + "queues": [], + "exchanges": [], + "bindings": [] + } diff --git a/deploy/k8s/redis-deployment.yaml b/deploy/k8s/redis-deployment.yaml new file mode 100644 index 0000000..b2f08fc --- /dev/null +++ b/deploy/k8s/redis-deployment.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis/redis-stack-server:latest + resources: + limits: + memory: "128Mi" + cpu: "500m" + ports: + - containerPort: 6379 diff --git a/deploy/k8s/redis-service.yaml b/deploy/k8s/redis-service.yaml new file mode 100644 index 0000000..ba82d15 --- /dev/null +++ b/deploy/k8s/redis-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-service +spec: + type: NodePort + selector: + app: redis + ports: + - port: 6379 + targetPort: 6379 + nodePort: 30001 + diff --git a/handler/async_task_cancel_handler.go b/handler/async_task_cancel_handler.go index 6fa0bc3..40fa480 100644 --- a/handler/async_task_cancel_handler.go +++ b/handler/async_task_cancel_handler.go @@ -7,6 +7,8 @@ import ( "modelRT/constants" "modelRT/database" "modelRT/logger" + "modelRT/mq" + "modelRT/mq/event" "modelRT/orm" "github.com/gin-gonic/gin" @@ -74,6 +76,10 @@ func AsyncTaskCancelHandler(c *gin.Context) { return } + if record, evtErr := event.NewTaskCancelledMessage(taskID.String(), string(asyncTask.TaskType)); evtErr == nil { + mq.TryEmitMessage(ctx, record) + } + err = database.UpdateAsyncTaskResultWithError(ctx, pgClient, taskID, 40009, "task cancelled by user", orm.JSONMap{ "cancelled_at": timestamp, "cancelled_by": "user", diff --git a/handler/async_task_create_handler.go b/handler/async_task_create_handler.go index 592460b..30b8531 100644 --- a/handler/async_task_create_handler.go +++ b/handler/async_task_create_handler.go @@ -5,6 +5,8 @@ import ( "modelRT/constants" "modelRT/database" "modelRT/logger" + "modelRT/mq" + "modelRT/mq/event" "modelRT/network" "modelRT/orm" "modelRT/task" @@ -77,6 +79,10 @@ func AsyncTaskCreateHandler(c *gin.Context) { task.TaskMsgChan <- msg logger.Info(ctx, "task enqueued to channel", "task_id", asyncTask.TaskID, "queue", constants.TaskQueueName) + if record, err := event.NewTaskSubmittedMessage(asyncTask.TaskID.String(), request.TaskType, 5); err == nil { + mq.TryEmitMessage(ctx, record) + } + logger.Info(ctx, "async task created success", "task_id", asyncTask.TaskID, "task_type", request.TaskType) // return success response diff --git a/main.go b/main.go index f120c6f..8034f87 100644 --- a/main.go +++ b/main.go @@ -188,6 +188,8 @@ func main() { // async push event to rabbitMQ go mq.PushUpDownLimitEventToRabbitMQ(ctx, mq.MsgChan) + // async push message (task state changes etc.) to rabbitMQ + go mq.PushMessageToRabbitMQ(ctx, mq.MessageMsgChan) // async push task message to rabbitMQ go task.PushTaskToRabbitMQ(ctx, modelRTConfig.RabbitMQConfig, task.TaskMsgChan) diff --git a/mq/emit.go b/mq/emit.go new file mode 100644 index 0000000..4c3dca6 --- /dev/null +++ b/mq/emit.go @@ -0,0 +1,28 @@ +// Package mq provides read or write access to message queue services +package mq + +import ( + "context" + + "modelRT/logger" + "modelRT/mq/event" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" +) + +// TryEmitMessage pushes a message record into MessageMsgChan non-blocking. +// If the channel is full the message is dropped and a warning is logged. +func TryEmitMessage(ctx context.Context, record *event.EventRecord) { + carrier := make(map[string]string) + otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(carrier)) + msg := &EventMessage{Record: record, TraceCarrier: carrier} + select { + case MessageMsgChan <- msg: + default: + logger.Warn(ctx, "message channel full, message dropped", + "event_uuid", record.EventUUID, + "category", record.Category, + ) + } +} diff --git a/mq/event/task_event_gen.go b/mq/event/task_event_gen.go new file mode 100644 index 0000000..ed32c4f --- /dev/null +++ b/mq/event/task_event_gen.go @@ -0,0 +1,81 @@ +// Package event define real time data evnet operation functions +package event + +import ( + "modelRT/constants" +) + +// NewTaskSubmittedMessage creates a message record for when a task is submitted to the queue +func NewTaskSubmittedMessage(taskID, taskType string, priority int) (*EventRecord, error) { + return NewPlatformEventRecord( + int(constants.EventGeneralPlatformSoft), + 0, + "async_task_submitted", + WithCategory(constants.MessageTaskSubmittedCategory), + WithCondition(map[string]any{ + "task_id": taskID, + "task_type": taskType, + "priority": priority, + }), + ) +} + +// NewTaskRunningMessage creates a message record for when a task begins execution +func NewTaskRunningMessage(taskID, taskType string) (*EventRecord, error) { + return NewPlatformEventRecord( + int(constants.EventGeneralPlatformSoft), + 0, + "async_task_running", + WithCategory(constants.MessageTaskRunningCategory), + WithCondition(map[string]any{ + "task_id": taskID, + "task_type": taskType, + }), + ) +} + +// NewTaskCompletedMessage creates a message record for when a task finishes successfully +func NewTaskCompletedMessage(taskID, taskType string, executionMs int64) (*EventRecord, error) { + return NewPlatformEventRecord( + int(constants.EventGeneralPlatformSoft), + 0, + "async_task_completed", + WithCategory(constants.MessageTaskCompletedCategory), + WithCondition(map[string]any{ + "task_id": taskID, + "task_type": taskType, + }), + WithResult(map[string]any{ + "execution_ms": executionMs, + }), + ) +} + +// NewTaskFailedMessage creates a message record for when a task fails during execution +func NewTaskFailedMessage(taskID, taskType, reason string) (*EventRecord, error) { + return NewPlatformEventRecord( + int(constants.EventGeneralPlatformSoft), + 0, + "async_task_failed", + WithCategory(constants.MessageTaskFailedCategory), + WithCondition(map[string]any{ + "task_id": taskID, + "task_type": taskType, + "reason": reason, + }), + ) +} + +// NewTaskCancelledMessage creates a message record for when a task is cancelled by a user +func NewTaskCancelledMessage(taskID, taskType string) (*EventRecord, error) { + return NewPlatformEventRecord( + int(constants.EventGeneralPlatformSoft), + 0, + "async_task_cancelled", + WithCategory(constants.MessageTaskCancelledCategory), + WithCondition(map[string]any{ + "task_id": taskID, + "task_type": taskType, + }), + ) +} diff --git a/mq/publish_message.go b/mq/publish_message.go new file mode 100644 index 0000000..46e796c --- /dev/null +++ b/mq/publish_message.go @@ -0,0 +1,149 @@ +// Package mq provides read or write access to message queue services +package mq + +import ( + "context" + "encoding/json" + "time" + + "modelRT/constants" + "modelRT/logger" + + amqp "github.com/rabbitmq/amqp091-go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" +) + +// MessageMsgChan buffers message records to be published to the message exchange asynchronously +var MessageMsgChan chan *EventMessage + +func init() { + MessageMsgChan = make(chan *EventMessage, 10000) +} + +func initMessageChannel(ctx context.Context) (*amqp.Channel, error) { + channel, err := GetConn().Channel() + if err != nil { + logger.Error(ctx, "open rabbitMQ server channel failed", "error", err) + return nil, err + } + + err = channel.ExchangeDeclare(constants.MessageDeadExchangeName, "topic", true, false, false, false, nil) + if err != nil { + logger.Error(ctx, "declare message dead letter exchange failed", "error", err) + return nil, err + } + + _, err = channel.QueueDeclare(constants.MessageDeadQueueName, true, false, false, false, nil) + if err != nil { + logger.Error(ctx, "declare message dead letter queue failed", "error", err) + return nil, err + } + + err = channel.QueueBind(constants.MessageDeadQueueName, constants.MessageDeadRoutingKey, constants.MessageDeadExchangeName, false, nil) + if err != nil { + logger.Error(ctx, "bind message dead letter queue failed", "error", err) + return nil, err + } + + err = channel.ExchangeDeclare(constants.MessageExchangeName, "topic", true, false, false, false, nil) + if err != nil { + logger.Error(ctx, "declare message exchange failed", "error", err) + return nil, err + } + + args := amqp.Table{ + "x-dead-letter-exchange": constants.MessageDeadExchangeName, + "x-dead-letter-routing-key": constants.MessageDeadRoutingKey, + } + _, err = channel.QueueDeclare(constants.MessageQueueName, true, false, false, false, args) + if err != nil { + logger.Error(ctx, "declare message queue failed", "error", err) + return nil, err + } + + err = channel.QueueBind(constants.MessageQueueName, constants.MessageRoutingKey, constants.MessageExchangeName, false, nil) + if err != nil { + logger.Error(ctx, "bind message queue failed", "error", err) + return nil, err + } + + if err := channel.Confirm(false); err != nil { + logger.Error(ctx, "channel could not be put into confirm mode", "error", err) + return nil, err + } + return channel, nil +} + +// PushMessageToRabbitMQ publishes message records from msgChan to the message exchange. +// The category of each record is used as the routing key so consumers can bind selectively. +func PushMessageToRabbitMQ(ctx context.Context, msgChan chan *EventMessage) { + channel, err := initMessageChannel(ctx) + if err != nil { + logger.Error(ctx, "initializing message rabbitMQ channel failed", "error", err) + return + } + + confirms := channel.NotifyPublish(make(chan amqp.Confirmation, 100)) + + go func() { + for { + select { + case confirm, ok := <-confirms: + if !ok { + return + } + if !confirm.Ack { + logger.Error(ctx, "publish message failed (rejected by rabbitMQ)", "tag", confirm.DeliveryTag) + } + case <-ctx.Done(): + return + } + } + }() + + for { + select { + case <-ctx.Done(): + logger.Info(ctx, "push message to rabbitMQ stopped by context cancel") + channel.Close() + return + case msg, ok := <-msgChan: + if !ok { + logger.Info(ctx, "push message to rabbitMQ stopped by msgChan closed") + channel.Close() + return + } + + record := msg.Record + + recordBytes, err := json.Marshal(record) + if err != nil { + logger.Error(ctx, "marshal message record failed", "event_uuid", record.EventUUID, "error", err) + continue + } + + msgCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(msg.TraceCarrier)) + headers := amqp.Table{} + otel.GetTextMapPropagator().Inject(msgCtx, amqpHeaderCarrier(headers)) + + routingKey := record.Category + pubCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + err = channel.PublishWithContext(pubCtx, + constants.MessageExchangeName, + routingKey, + false, + false, + amqp.Publishing{ + ContentType: "text/plain", + Body: recordBytes, + Headers: headers, + }) + cancel() + + if err != nil { + logger.Error(ctx, "publish message to rabbitMQ failed", "routing_key", routingKey, "error", err) + } + } + } +} diff --git a/task/worker.go b/task/worker.go index b51e00d..1febca4 100644 --- a/task/worker.go +++ b/task/worker.go @@ -13,6 +13,7 @@ import ( "modelRT/database" "modelRT/logger" "modelRT/mq" + "modelRT/mq/event" "modelRT/orm" "github.com/gofrs/uuid" @@ -314,6 +315,10 @@ func (w *TaskWorker) handleMessage(msg amqp.Delivery) { return } + if record, err := event.NewTaskRunningMessage(taskMsg.TaskID.String(), string(taskMsg.TaskType)); err == nil { + mq.TryEmitMessage(ctx, record) + } + // Execute task using handler startTime := time.Now() err := w.dispatch(ctx, taskMsg.TaskType, taskMsg.TaskID, taskMsg.Params, &msg) @@ -332,6 +337,10 @@ func (w *TaskWorker) handleMessage(msg amqp.Delivery) { logger.Error(ctx, "Failed to update task with error", "error", updateErr) } + if record, recErr := event.NewTaskFailedMessage(taskMsg.TaskID.String(), string(taskMsg.TaskType), err.Error()); recErr == nil { + mq.TryEmitMessage(ctx, record) + } + // Ack message even if task failed (we don't want to retry indefinitely) msg.Ack(false) w.metrics.mu.Lock() @@ -347,6 +356,10 @@ func (w *TaskWorker) handleMessage(msg amqp.Delivery) { // Still ack the message since task was processed successfully } + if record, err := event.NewTaskCompletedMessage(taskMsg.TaskID.String(), string(taskMsg.TaskType), processingTime.Milliseconds()); err == nil { + mq.TryEmitMessage(ctx, record) + } + // Acknowledge message msg.Ack(false) From d051c161b7197d1269c0419a0b9c58b76484e807 Mon Sep 17 00:00:00 2001 From: douxu Date: Mon, 18 May 2026 16:49:46 +0800 Subject: [PATCH 38/43] perf: parallelize GetFullMeasurementSet with errgroup - run 5 independent DB queries concurrently via errgroup.WithContext - add ctx parameter and bind db with WithContext for cancellation support - replace silent error swallowing (if err == nil) with wrapped error returns - promote golang.org/x/sync to direct dependency in go.mod --- database/query_component_measurement.go | 121 +++++++++++++++--------- go.mod | 2 +- main.go | 2 +- 3 files changed, 77 insertions(+), 48 deletions(-) diff --git a/database/query_component_measurement.go b/database/query_component_measurement.go index b8ddb7d..e9e95ca 100644 --- a/database/query_component_measurement.go +++ b/database/query_component_measurement.go @@ -2,8 +2,12 @@ package database import ( + "context" + "fmt" + "modelRT/orm" + "golang.org/x/sync/errgroup" "gorm.io/gorm" ) @@ -17,7 +21,7 @@ type StationWithParent struct { ZoneTag string `gorm:"column:zone_tag"` } -func GetFullMeasurementSet(db *gorm.DB) (*orm.MeasurementSet, error) { +func GetFullMeasurementSet(ctx context.Context, db *gorm.DB) (*orm.MeasurementSet, error) { mSet := &orm.MeasurementSet{ GridToZoneTags: make(map[string][]string), ZoneToStationTags: make(map[string][]string), @@ -26,85 +30,110 @@ func GetFullMeasurementSet(db *gorm.DB) (*orm.MeasurementSet, error) { CompTagToMeasTags: make(map[string][]string), } - var grids []orm.Grid - if err := db.Table("grid").Select("tagname").Scan(&grids).Error; err == nil { - for _, g := range grids { - if g.TAGNAME != "" { - mSet.AllGridTags = append(mSet.AllGridTags, g.TAGNAME) + g, gctx := errgroup.WithContext(ctx) + db = db.WithContext(gctx) + + g.Go(func() error { + var grids []orm.Grid + if err := db.Table("grid").Select("tagname").Scan(&grids).Error; err != nil { + return fmt.Errorf("query grids: %w", err) + } + for _, grid := range grids { + if grid.TAGNAME != "" { + mSet.AllGridTags = append(mSet.AllGridTags, grid.TAGNAME) } } - } + return nil + }) - var zones []struct { - orm.Zone - GridTag string `gorm:"column:grid_tag"` - } - if err := db.Table("zone"). - Select("zone.*, grid.tagname as grid_tag"). - Joins("left join grid on zone.grid_id = grid.id"). - Scan(&zones).Error; err == nil { + g.Go(func() error { + var zones []struct { + orm.Zone + GridTag string `gorm:"column:grid_tag"` + } + if err := db.Table("zone"). + Select("zone.*, grid.tagname as grid_tag"). + Joins("left join grid on zone.grid_id = grid.id"). + Scan(&zones).Error; err != nil { + return fmt.Errorf("query zones: %w", err) + } for _, z := range zones { mSet.AllZoneTags = append(mSet.AllZoneTags, z.TAGNAME) if z.GridTag != "" { mSet.GridToZoneTags[z.GridTag] = append(mSet.GridToZoneTags[z.GridTag], z.TAGNAME) } } - } + return nil + }) - var stations []struct { - orm.Station - ZoneTag string `gorm:"column:zone_tag"` - } - if err := db.Table("station"). - Select("station.*, zone.tagname as zone_tag"). - Joins("left join zone on station.zone_id = zone.id"). - Scan(&stations).Error; err == nil { + g.Go(func() error { + var stations []struct { + orm.Station + ZoneTag string `gorm:"column:zone_tag"` + } + if err := db.Table("station"). + Select("station.*, zone.tagname as zone_tag"). + Joins("left join zone on station.zone_id = zone.id"). + Scan(&stations).Error; err != nil { + return fmt.Errorf("query stations: %w", err) + } for _, s := range stations { mSet.AllStationTags = append(mSet.AllStationTags, s.TAGNAME) if s.ZoneTag != "" { mSet.ZoneToStationTags[s.ZoneTag] = append(mSet.ZoneToStationTags[s.ZoneTag], s.TAGNAME) } } - } + return nil + }) - var comps []struct { - orm.Component - StationTag string `gorm:"column:station_tag"` - } - if err := db.Table("component"). - Select("component.*, station.tagname as station_tag"). - Joins("left join station on component.station_id = station.id"). - Scan(&comps).Error; err == nil { + g.Go(func() error { + var comps []struct { + orm.Component + StationTag string `gorm:"column:station_tag"` + } + if err := db.Table("component"). + Select("component.*, station.tagname as station_tag"). + Joins("left join station on component.station_id = station.id"). + Scan(&comps).Error; err != nil { + return fmt.Errorf("query components: %w", err) + } for _, c := range comps { mSet.AllCompNSPaths = append(mSet.AllCompNSPaths, c.NSPath) mSet.AllCompTags = append(mSet.AllCompTags, c.Tag) - if c.StationTag != "" { mSet.StationToCompNSPaths[c.StationTag] = append(mSet.StationToCompNSPaths[c.StationTag], c.NSPath) } - if c.NSPath != "" { mSet.CompNSPathToCompTags[c.NSPath] = append(mSet.CompNSPathToCompTags[c.NSPath], c.Tag) } } - } + return nil + }) - mSet.AllConfigTags = append(mSet.AllConfigTags, "bay") - - var measurements []struct { - orm.Measurement - CompTag string `gorm:"column:comp_tag"` - } - if err := db.Table("measurement"). - Select("measurement.*, component.tag as comp_tag"). - Joins("left join component on measurement.component_uuid = component.global_uuid"). - Scan(&measurements).Error; err == nil { + g.Go(func() error { + var measurements []struct { + orm.Measurement + CompTag string `gorm:"column:comp_tag"` + } + if err := db.Table("measurement"). + Select("measurement.*, component.tag as comp_tag"). + Joins("left join component on measurement.component_uuid = component.global_uuid"). + Scan(&measurements).Error; err != nil { + return fmt.Errorf("query measurements: %w", err) + } for _, m := range measurements { mSet.AllMeasTags = append(mSet.AllMeasTags, m.Tag) if m.CompTag != "" { mSet.CompTagToMeasTags[m.CompTag] = append(mSet.CompTagToMeasTags[m.CompTag], m.Tag) } } + return nil + }) + + if err := g.Wait(); err != nil { + return nil, err } + + mSet.AllConfigTags = append(mSet.AllConfigTags, "bay") return mSet, nil } diff --git a/go.mod b/go.mod index 9f24f78..3e33ddf 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 go.uber.org/zap v1.27.0 + golang.org/x/sync v0.20.0 gorm.io/driver/mysql v1.5.7 gorm.io/driver/postgres v1.5.9 gorm.io/gorm v1.25.12 @@ -94,7 +95,6 @@ require ( golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.52.0 // indirect - golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.35.0 // indirect golang.org/x/tools v0.42.0 // indirect diff --git a/main.go b/main.go index 8034f87..ef62aae 100644 --- a/main.go +++ b/main.go @@ -214,7 +214,7 @@ func main() { panic(err) } - measurementSet, err := database.GetFullMeasurementSet(tx) + measurementSet, err := database.GetFullMeasurementSet(ctx, tx) if err != nil { logger.Error(ctx, "generate component measurement group failed", "error", err) panic(err) From 4a2666aa3b53522dd484fc250b47c214b1ecc914 Mon Sep 17 00:00:00 2001 From: douxu Date: Tue, 19 May 2026 17:38:22 +0800 Subject: [PATCH 39/43] fix: correct caller frames in GORM logger and DB arg in main - add *Skip variants (logSkip, makeLogFieldsSkip, getLoggerCallerInfoSkip) so wrapper functions report the true call site, not logger internals - switch GormLogger.Trace to use ErrorSkip/WarnSkip/InfoSkip with extraSkip=1 so SQL log lines point to GORM caller rather than the logger facade - pass postgresDBClient instead of tx to GetFullMeasurementSet in main --- logger/facede.go | 21 ++++++++++++++++++++- logger/gorm_logger.go | 9 ++++----- logger/logger.go | 16 ++++++++++++---- main.go | 2 +- 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/logger/facede.go b/logger/facede.go index b9a5b54..1533cdf 100644 --- a/logger/facede.go +++ b/logger/facede.go @@ -39,11 +39,30 @@ func Error(ctx context.Context, msg string, kv ...any) { } func (f *facade) log(ctx context.Context, lvl zapcore.Level, msg string, kv ...any) { - fields := makeLogFields(ctx, kv...) + f.logSkip(ctx, lvl, 0, msg, kv...) +} + +func (f *facade) logSkip(ctx context.Context, lvl zapcore.Level, extraSkip int, msg string, kv ...any) { + fields := makeLogFieldsSkip(ctx, extraSkip, kv...) ce := f._logger.Check(lvl, msg) ce.Write(fields...) } +// ErrorSkip logs at error level with extra caller skip frames for wrapper functions. +func ErrorSkip(ctx context.Context, extraSkip int, msg string, kv ...any) { + logFacade().logSkip(ctx, zapcore.ErrorLevel, extraSkip, msg, kv...) +} + +// WarnSkip logs at warn level with extra caller skip frames for wrapper functions. +func WarnSkip(ctx context.Context, extraSkip int, msg string, kv ...any) { + logFacade().logSkip(ctx, zapcore.WarnLevel, extraSkip, msg, kv...) +} + +// InfoSkip logs at info level with extra caller skip frames for wrapper functions. +func InfoSkip(ctx context.Context, extraSkip int, msg string, kv ...any) { + logFacade().logSkip(ctx, zapcore.InfoLevel, extraSkip, msg, kv...) +} + func logFacade() *facade { fOnce.Do(func() { f = &facade{ diff --git a/logger/gorm_logger.go b/logger/gorm_logger.go index c5e6c01..fadd2e6 100644 --- a/logger/gorm_logger.go +++ b/logger/gorm_logger.go @@ -48,14 +48,13 @@ func (l *GormLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql duration := time.Since(begin).Milliseconds() // get gorm exec sql and rows affected sql, rows := fc() - // gorm error judgment if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - Error(ctx, "SQL ERROR", "sql", sql, "rows", rows, "dur(ms)", duration) + ErrorSkip(ctx, 1, "SQL ERROR", "sql", sql, "rows", rows, "dur(ms)", duration) + return } - // slow query judgment if duration > l.SlowThreshold.Milliseconds() { - Warn(ctx, "SQL SLOW", "sql", sql, "rows", rows, "dur(ms)", duration) + WarnSkip(ctx, 1, "SQL SLOW", "sql", sql, "rows", rows, "dur(ms)", duration) } else { - Info(ctx, "SQL INFO", "sql", sql, "rows", rows, "dur(ms)", duration) + InfoSkip(ctx, 1, "SQL INFO", "sql", sql, "rows", rows, "dur(ms)", duration) } } diff --git a/logger/logger.go b/logger/logger.go index 3b4175f..f7fa247 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -47,7 +47,10 @@ func (l *logger) log(lvl zapcore.Level, msg string, kv ...any) { } func makeLogFields(ctx context.Context, kv ...any) []zap.Field { - // Ensure that log information appears in pairs in the form of key-value pairs + return makeLogFieldsSkip(ctx, 0, kv...) +} + +func makeLogFieldsSkip(ctx context.Context, extraSkip int, kv ...any) []zap.Field { if len(kv)%2 != 0 { kv = append(kv, "unknown") } @@ -57,7 +60,7 @@ func makeLogFields(ctx context.Context, kv ...any) []zap.Field { spanID := spanCtx.SpanID().String() kv = append(kv, "traceID", traceID, "spanID", spanID) - funcName, file, line := getLoggerCallerInfo() + funcName, file, line := getLoggerCallerInfoSkip(extraSkip) kv = append(kv, "func", funcName, "file", file, "line", line) fields := make([]zap.Field, 0, len(kv)/2) for i := 0; i < len(kv); i += 2 { @@ -85,9 +88,14 @@ func makeLogFields(ctx context.Context, kv ...any) []zap.Field { return fields } -// getLoggerCallerInfo define func of return log caller information、method name、file name、line number +// getLoggerCallerInfo returns caller info at a fixed depth for the standard facade call chain. func getLoggerCallerInfo() (funcName, file string, line int) { - pc, file, line, ok := runtime.Caller(4) + return getLoggerCallerInfoSkip(0) +} + +// getLoggerCallerInfoSkip returns caller info with additional skip frames beyond the standard depth. +func getLoggerCallerInfoSkip(extraSkip int) (funcName, file string, line int) { + pc, file, line, ok := runtime.Caller(4 + extraSkip) if !ok { return } diff --git a/main.go b/main.go index ef62aae..259b269 100644 --- a/main.go +++ b/main.go @@ -214,7 +214,7 @@ func main() { panic(err) } - measurementSet, err := database.GetFullMeasurementSet(ctx, tx) + measurementSet, err := database.GetFullMeasurementSet(ctx, postgresDBClient) if err != nil { logger.Error(ctx, "generate component measurement group failed", "error", err) panic(err) From 57371fbf1f4714c895536510f83db3a51d59e6f2 Mon Sep 17 00:00:00 2001 From: douxu Date: Wed, 27 May 2026 16:51:00 +0800 Subject: [PATCH 40/43] docs: add Minikube PostgreSQL manifests and clean deploy markdown - add split PostgreSQL K8s manifests for ConfigMap, Service, PVC, and StatefulSet - expose PostgreSQL through NodePort for local Minikube access - replace deploy.md LaTeX text syntax with Markdown inline code formatting - keep deployment documentation rendering stable in Wiki.js and Markdown viewers --- deploy/deploy.md | 70 +++++++++++++++++----------------- deploy/k8s/pg-configmap.yaml | 8 ++++ deploy/k8s/pg-pvc.yaml | 10 +++++ deploy/k8s/pg-service.yaml | 15 ++++++++ deploy/k8s/pg-statefulset.yaml | 61 +++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 35 deletions(-) create mode 100644 deploy/k8s/pg-configmap.yaml create mode 100644 deploy/k8s/pg-pvc.yaml create mode 100644 deploy/k8s/pg-service.yaml create mode 100644 deploy/k8s/pg-statefulset.yaml diff --git a/deploy/deploy.md b/deploy/deploy.md index 8930c91..0d89a28 100644 --- a/deploy/deploy.md +++ b/deploy/deploy.md @@ -1,12 +1,12 @@ # 项目依赖服务部署指南 -本项目依赖于 $\text{PostgreSQL}$ 数据库和 $\text{Redis Stack Server}$（包含 $\text{Redisearch}$ 等模块）部署文档将使用 $\text{Docker}$ 容器化技术部署这两个依赖服务 +本项目依赖于 `PostgreSQL` 数据库和 `Redis Stack Server`（包含 `Redisearch` 等模块）部署文档将使用 `Docker` 容器化技术部署这两个依赖服务 ## 前提条件 -1. 已安装 $\text{Docker}$ +1. 已安装 `Docker` 2. 下载相关容器镜像 -3. 确保主机的 $\text{5432}$ 端口（$\text{Postgres}$）和 $\text{6379}$ 端口（$\text{Redis}$）未被占用 +3. 确保主机的 `5432` 端口（`Postgres`）和 `6379` 端口（`Redis`）未被占用 ### 1\. 部署 PostgreSQL 数据库 @@ -14,7 +14,7 @@ #### 1.1 部署命令 -运行以下命令启动 $\text{PostgreSQL}$ 容器 +运行以下命令启动 `PostgreSQL` 容器 ```bash docker run --name postgres \ @@ -47,7 +47,7 @@ docker logs postgres #### 1.4 初始化异步任务表 -$\text{PostgreSQL}$ 启动后执行以下建表语句，创建异步任务系统所需的两张表： +`PostgreSQL` 启动后执行以下建表语句，创建异步任务系统所需的两张表： ```sql -- ========================================== @@ -109,11 +109,11 @@ COMMENT ON TABLE async_task_result IS '异步任务执行结果表'; ### 2\. 部署 Redis Stack Server -我们将使用 `redis/redis-stack-server:latest` 镜像该镜像内置了 $\text{Redisearch}$ 模块，用于 $\text{ModelRT}$ 项目中补全功能 +我们将使用 `redis/redis-stack-server:latest` 镜像该镜像内置了 `Redisearch` 模块，用于 `ModelRT` 项目中补全功能 #### 2.1 部署命令 -运行以下命令启动 $\text{Redis Stack Server}$ 容器 +运行以下命令启动 `Redis Stack Server` 容器 ```bash docker run --name redis -p 6379:6379 \ @@ -130,7 +130,7 @@ docker run --name redis -p 6379:6379 \ | **地址** | `localhost:6379` | | | **密码** | **无** | 默认未设置密码 | -> **注意:** 生产环境中建议使用 `-e REDIS_PASSWORD=` 参数来设置 $\text{Redis}$ 访问密码 +> **注意:** 生产环境中建议使用 `-e REDIS_PASSWORD=` 参数来设置 `Redis` 访问密码 #### 2.3 状态检查 @@ -403,46 +403,46 @@ go run deploy/redis-test-data/measurments-recommend/measurement_injection.go | 类别 | 参数名 | 作用描述 | 示例值 | | :--- | :--- | :--- | :--- | -| **Postgres** | `host` | PostgreSQL 数据库服务器的 $\text{IP}$ 地址或域名。 | `"192.168.1.101"` | +| **Postgres** | `host` | PostgreSQL 数据库服务器的 `IP` 地址或域名。 | `"192.168.1.101"` | | | `port` | PostgreSQL 数据库服务器的端口号。 | `5432` | | | `database` | 连接的数据库名称。 | `"demo"` | | | `user` | 连接数据库所使用的用户名。 | `"postgres"` | | | `password` | 连接数据库所使用的密码。 | `"coslight"` | -| **Kafka** | `servers` | Kafka 集群的 $\text{Bootstrap Server}$ 地址列表（通常是 $\text{host:port}$ 形式，多个地址用逗号分隔）。 | `"localhost:9092"` | +| **Kafka** | `servers` | Kafka 集群的 `Bootstrap Server` 地址列表（通常是 `host:port` 形式，多个地址用逗号分隔）。 | `"localhost:9092"` | | | `port` | Kafka 服务器的端口号。 | `9092` | -| | `group_id` | 消费者组 $\text{ID}$，用于标识和管理一组相关的消费者。 | `"modelRT"` | +| | `group_id` | 消费者组 `ID`，用于标识和管理一组相关的消费者。 | `"modelRT"` | | | `topic` | Kafka 消息的主题名称。 | `""` | -| | `auto_offset_reset` | 消费者首次启动或 $\text{Offset}$ 无效时，从哪个位置开始消费（如 `earliest` 或 `latest`）。 | `"earliest"` | -| | `enable_auto_commit` | 是否自动提交 $\text{Offset}$。设为 $\text{false}$ 通常用于手动控制 $\text{Offset}$ 提交。 | `"false"` | +| | `auto_offset_reset` | 消费者首次启动或 `Offset` 无效时，从哪个位置开始消费（如 `earliest` 或 `latest`）。 | `"earliest"` | +| | `enable_auto_commit` | 是否自动提交 `Offset`。设为 `false` 通常用于手动控制 `Offset` 提交。 | `"false"` | | | `read_message_time_duration` | 读取消息时的超时或等待时间。 | `”0.5s"` | | **Logger (Zap)** | `mode` | 日志模式，通常为 `development`（开发）或 `production`（生产）。影响日志格式。 | `"development"` | -| | `level` | 最低日志级别（如 $\text{debug, info, warn, error}$）。 | `"debug"` | +| | `level` | 最低日志级别（如 `debug`, `info`, `warn`, `error`）。 | `"debug"` | | | `filepath` | 日志文件的输出路径和名称格式（`%s` 会被替换为日期等）。 | `"/Users/douxu/Workspace/coslight/modelRT/modelRT-%s.log"` | -| | `maxsize` | 单个日志文件最大大小（单位：$\text{MB}$）。 | `1` | +| | `maxsize` | 单个日志文件最大大小（单位：`MB`）。 | `1` | | | `maxbackups` | 保留旧日志文件的最大个数。 | `5` | | | `maxage` | 保留旧日志文件的最大天数。 | `30` | | | `compress` | 是否压缩备份的日志文件。 | `false` | | **Ants Pool** | `parse_concurrent_quantity` | 用于解析任务的协程池最大并发数量。 | `10` | | | `rtd_receive_concurrent_quantity` | 用于实时数据接收任务的协程池最大并发数量。 | `10` | -| **Locker Redis** | `addr` | 分布式锁服务所使用的 $\text{Redis}$ 地址。 | `"127.0.0.1:6379"` | -| | `password` | $\text{Locker Redis}$ 的密码。 | `""` | -| | `db` | $\text{Locker Redis}$ 使用的数据库编号。 | `1` | -| | `poolsize` | $\text{Locker Redis}$ 连接池的最大连接数。 | `50` | -| | `timeout` | $\text{Locker Redis}$ 连接操作的超时时间（单位：毫秒）。 | `10` | -| **Storage Redis** | `addr` | 数据存储服务所使用的 $\text{Redis}$ 地址（例如 $\text{Redisearch}$）。 | `"127.0.0.1:6379"` | -| | `password` | $\text{Storage Redis}$ 的密码。 | `""` | -| | `db` | $\text{Storage Redis}$ 使用的数据库编号。 | `0` | -| | `poolsize` | $\text{Storage Redis}$ 连接池的最大连接数。 | `50` | -| | `timeout` | $\text{Storage Redis}$ 连接操作的超时时间（单位：毫秒）。 | `10` | -| **Base Config** | `grid_id` | 项目所操作的默认电网 $\text{ID}$。 | `1` | -| | `zone_id` | 项目所操作的默认区域 $\text{ID}$。 | `1` | -| | `station_id` | 项目所操作的默认变电站 $\text{ID}$。 | `1` | +| **Locker Redis** | `addr` | 分布式锁服务所使用的 `Redis` 地址。 | `"127.0.0.1:6379"` | +| | `password` | `Locker Redis` 的密码。 | `""` | +| | `db` | `Locker Redis` 使用的数据库编号。 | `1` | +| | `poolsize` | `Locker Redis` 连接池的最大连接数。 | `50` | +| | `timeout` | `Locker Redis` 连接操作的超时时间（单位：毫秒）。 | `10` | +| **Storage Redis** | `addr` | 数据存储服务所使用的 `Redis` 地址（例如 `Redisearch`）。 | `"127.0.0.1:6379"` | +| | `password` | `Storage Redis` 的密码。 | `""` | +| | `db` | `Storage Redis` 使用的数据库编号。 | `0` | +| | `poolsize` | `Storage Redis` 连接池的最大连接数。 | `50` | +| | `timeout` | `Storage Redis` 连接操作的超时时间（单位：毫秒）。 | `10` | +| **Base Config** | `grid_id` | 项目所操作的默认电网 `ID`。 | `1` | +| | `zone_id` | 项目所操作的默认区域 `ID`。 | `1` | +| | `station_id` | 项目所操作的默认变电站 `ID`。 | `1` | | **Service Config** | `service_name` | 服务名称，用于日志、监控等标识。 | `"modelRT"` | | | `secret_key` | 服务内部使用的秘钥，用于签名或认证。 | `"modelrt_key"` | -| **DataRT API** | `host` | 外部 $\text{DataRT}$ 服务的主机地址。 | `"http://127.0.0.1"` | -| | `port` | $\text{DataRT}$ 服务的端口号。 | `8888` | -| | `polling_api` | 轮询数据的 $\text{API}$ 路径。 | `"datart/getPointData"` | -| | `polling_api_method` | 调用该 $\text{API}$ 使用的 $\text{HTTP}$ 方法。 | `"GET"` | +| **DataRT API** | `host` | 外部 `DataRT` 服务的主机地址。 | `"http://127.0.0.1"` | +| | `port` | `DataRT` 服务的端口号。 | `8888` | +| | `polling_api` | 轮询数据的 `API` 路径。 | `"datart/getPointData"` | +| | `polling_api_method` | 调用该 `API` 使用的 `HTTP` 方法。 | `"GET"` | #### 3.2 编译 ModelRT 服务 @@ -762,7 +762,7 @@ kubectl delete secret modelrt-certs ### 6\. 部署可观测性栈（Kubernetes） -在 $\text{Kubernetes}$ 集群中部署 $\text{Jaeger}$（链路追踪）+ $\text{Loki + Promtail + Grafana}$（日志可视化）。所有资源部署在 `default` 命名空间，$\text{YAML}$ 文件位于 `deploy/k8s/`。 +在 `Kubernetes` 集群中部署 `Jaeger`（链路追踪）+ `Loki + Promtail + Grafana`（日志可视化）。所有资源部署在 `default` 命名空间，`YAML` 文件位于 `deploy/k8s/`。 #### 6.1 部署 Jaeger @@ -841,7 +841,7 @@ kubectl delete -f deploy/k8s/ ### 7\. Mac 本地访问（SSH 隧道） -$\text{ModelRT / EventRT}$ 在 $\text{Mac}$ 本地运行时，依赖的 $\text{RabbitMQ}$、$\text{Redis}$、$\text{Jaeger}$、$\text{Loki}$、$\text{Grafana}$ 均部署在 $\text{Ubuntu}$ 宿主机（`192.168.1.101`）上的 $\text{Minikube}$（`192.168.49.2`）中。由于 $\text{Minikube}$ 网络不直接对外暴露，需通过 $\text{SSH}$ 本地端口转发建立访问隧道。 +`ModelRT / EventRT` 在 `Mac` 本地运行时，依赖的 `RabbitMQ`、`Redis`、`Jaeger`、`Loki`、`Grafana` 均部署在 `Ubuntu` 宿主机（`192.168.1.101`）上的 `Minikube`（`192.168.49.2`）中。由于 `Minikube` 网络不直接对外暴露，需通过 `SSH` 本地端口转发建立访问隧道。 #### 7.1 网络拓扑 @@ -888,7 +888,7 @@ ssh -fN \ | `3100` | `31100` | Loki | 日志查询 API | | `3000` | `31000` | Grafana | 可视化界面 `http://localhost:3000` | -> **注意：** 隧道建立后，本地配置文件中所有服务地址均填 `localhost:<本地端口>`，无需修改即可在 $\text{Mac}$ 上直接运行服务。 +> **注意：** 隧道建立后，本地配置文件中所有服务地址均填 `localhost:<本地端口>`，无需修改即可在 `Mac` 上直接运行服务。 #### 7.4 关闭隧道 diff --git a/deploy/k8s/pg-configmap.yaml b/deploy/k8s/pg-configmap.yaml new file mode 100644 index 0000000..1daf181 --- /dev/null +++ b/deploy/k8s/pg-configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-config +data: + POSTGRES_DB: demo + POSTGRES_USER: postgres + POSTGRES_PASSWORD: coslight diff --git a/deploy/k8s/pg-pvc.yaml b/deploy/k8s/pg-pvc.yaml new file mode 100644 index 0000000..f172ce7 --- /dev/null +++ b/deploy/k8s/pg-pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/deploy/k8s/pg-service.yaml b/deploy/k8s/pg-service.yaml new file mode 100644 index 0000000..5e11fe0 --- /dev/null +++ b/deploy/k8s/pg-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres + labels: + app: postgres +spec: + type: NodePort + selector: + app: postgres + ports: + - name: postgres + port: 5432 + targetPort: 5432 + nodePort: 30432 diff --git a/deploy/k8s/pg-statefulset.yaml b/deploy/k8s/pg-statefulset.yaml new file mode 100644 index 0000000..bdf0ec8 --- /dev/null +++ b/deploy/k8s/pg-statefulset.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + labels: + app: postgres +spec: + serviceName: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:13.16 + imagePullPolicy: IfNotPresent + ports: + - name: postgres + containerPort: 5432 + envFrom: + - configMapRef: + name: postgres-config + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + readinessProbe: + exec: + command: + - sh + - -c + - pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" + initialDelaySeconds: 8 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 12 + livenessProbe: + exec: + command: + - sh + - -c + - pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: postgres-data From 9c4dcd29e48949687b2046ad2f8207aa801382b4 Mon Sep 17 00:00:00 2001 From: douxu Date: Thu, 28 May 2026 16:36:51 +0800 Subject: [PATCH 41/43] chore: bump Go to 1.26.3 - upgrade go directive in go.mod from 1.25.0 to 1.26.3 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3e33ddf..441ab18 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module modelRT -go 1.25.0 +go 1.26.3 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 From bacd43617ec88850e52ec06780751c965f70c6c8 Mon Sep 17 00:00:00 2001 From: douxu Date: Fri, 29 May 2026 10:56:17 +0800 Subject: [PATCH 42/43] chore: bind sensitive config to env vars and bump Go image to 1.25 - bind postgres.password to POSTGRES_PASSWORD env var via viper BindEnv - bind service.secret_key to SERVICE_SECRET_KEY env var via viper BindEnv - upgrade builder base image from golang:1.24-alpine to golang:1.25-alpine --- config/config.go | 3 +++ deploy/dockerfile/modelrt.Dockerfile | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 5a87f34..a0ddc7f 100644 --- a/config/config.go +++ b/config/config.go @@ -145,6 +145,9 @@ func ReadAndInitConfig(configDir, configName, configType string) (modelRTConfig panic(err) } + config.BindEnv("postgres.password", "POSTGRES_PASSWORD") + config.BindEnv("service.secret_key", "SERVICE_SECRET_KEY") + if err := config.Unmarshal(&modelRTConfig); err != nil { panic(fmt.Sprintf("unmarshal modelRT config failed:%s\n", err.Error())) } diff --git a/deploy/dockerfile/modelrt.Dockerfile b/deploy/dockerfile/modelrt.Dockerfile index c091aae..4f83bb4 100644 --- a/deploy/dockerfile/modelrt.Dockerfile +++ b/deploy/dockerfile/modelrt.Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24-alpine AS builder +FROM golang:1.25-alpine AS builder RUN apk --no-cache upgrade WORKDIR /app From 57d1111a83a05e8f6b442ba91e961e992f24f773 Mon Sep 17 00:00:00 2001 From: douxu Date: Fri, 29 May 2026 14:28:58 +0800 Subject: [PATCH 43/43] refactor: modernize Go idioms and add MongoDB K8s manifests - replace interface{} with any across ~30 files for Go 1.18+ style - adopt for-range-over-int loops in place of explicit index loops - use maps.Copy from stdlib to replace manual map copy loops - use min() builtin for exponential backoff delay cap in retry_manager - add MongoDB 7.0 K8s manifests (StatefulSet, Service, PVC, Secret) - document PostgreSQL and MongoDB deploy steps in deploy.md with SSH tunnel port mappings --- common/errcode/error.go | 8 +- config/anchor_param_config.go | 2 +- deploy/deploy.md | 112 +++++++++++++++++- deploy/k8s/mongodb-pvc.yaml | 10 ++ deploy/k8s/mongodb-secret.yaml | 8 ++ deploy/k8s/mongodb-service.yaml | 15 +++ deploy/k8s/mongodb-statefulset.yaml | 61 ++++++++++ .../real-time-subpull/sub_data_injection.go | 4 +- diagram/hash_test.go | 2 +- diagram/multi_branch_tree.go | 2 +- diagram/redis_hash.go | 4 +- diagram/redis_string.go | 2 +- diagram/redis_zset.go | 2 +- diagram/topologic_set.go | 2 +- handler/anchor_point_replace.go | 2 +- handler/async_task_create_handler.go | 2 +- handler/attr_delete.go | 4 +- handler/attr_load.go | 4 +- handler/attr_update.go | 4 +- handler/circuit_diagram_create.go | 12 +- handler/circuit_diagram_delete.go | 16 +-- handler/circuit_diagram_load.go | 14 +-- handler/circuit_diagram_update.go | 14 +-- handler/history_data_query.go | 2 +- handler/real_time_data_receive.go | 6 +- logger/loki_syncer.go | 5 +- middleware/trace.go | 2 +- model/attribute_model.go | 2 +- network/api_endpoint.go | 2 +- network/async_task_request.go | 22 ++-- network/attr_request.go | 4 +- network/request_convert.go | 2 +- pool/concurrency_anchor_parse.go | 2 +- pool/concurrency_model_parse.go | 2 +- real-time-data/cache.go | 12 +- task/base_task.go | 4 +- task/retry_manager.go | 25 ++-- task/test_task.go | 8 +- task/worker.go | 17 +-- 39 files changed, 307 insertions(+), 116 deletions(-) create mode 100644 deploy/k8s/mongodb-pvc.yaml create mode 100644 deploy/k8s/mongodb-secret.yaml create mode 100644 deploy/k8s/mongodb-service.yaml create mode 100644 deploy/k8s/mongodb-statefulset.yaml diff --git a/common/errcode/error.go b/common/errcode/error.go index 18eca89..1d12276 100644 --- a/common/errcode/error.go +++ b/common/errcode/error.go @@ -139,10 +139,10 @@ func (e *AppError) SetMsg(msg string) *AppError { } type formattedErr struct { - Code int `json:"code"` - Msg string `json:"msg"` - Cause interface{} `json:"cause"` - Occurred string `json:"occurred"` + Code int `json:"code"` + Msg string `json:"msg"` + Cause any `json:"cause"` + Occurred string `json:"occurred"` } // toStructuredError define func convert AppError to structured error for better readability diff --git a/config/anchor_param_config.go b/config/anchor_param_config.go index bd9d7f3..b5e201c 100644 --- a/config/anchor_param_config.go +++ b/config/anchor_param_config.go @@ -42,7 +42,7 @@ var baseCurrentFunc = func(archorValue float64, args ...float64) float64 { } // SelectAnchorCalculateFuncAndParams define select anchor func and anchor calculate value by component type 、 anchor name and component data -func SelectAnchorCalculateFuncAndParams(componentType int, anchorName string, componentData map[string]interface{}) (func(archorValue float64, args ...float64) float64, []float64) { +func SelectAnchorCalculateFuncAndParams(componentType int, anchorName string, componentData map[string]any) (func(archorValue float64, args ...float64) float64, []float64) { if componentType == constants.DemoType { switch anchorName { case "voltage": diff --git a/deploy/deploy.md b/deploy/deploy.md index 0d89a28..7b758b0 100644 --- a/deploy/deploy.md +++ b/deploy/deploy.md @@ -1,4 +1,4 @@ -# 项目依赖服务部署指南 +# 项目服务部署指南 本项目依赖于 `PostgreSQL` 数据库和 `Redis Stack Server`（包含 `Redisearch` 等模块）部署文档将使用 `Docker` 容器化技术部署这两个依赖服务 @@ -679,6 +679,108 @@ kubectl apply -f deploy/k8s/rabbitmq-service.yaml > **注意：** 证书认证用户的 `password_hash` 留空；RabbitMQ 通过 `ssl_cert_login_from = common_name` 将证书 CN 映射为用户名。 +#### 4.4 部署 PostgreSQL + +```bash +kubectl apply -f deploy/k8s/pg-configmap.yaml +kubectl apply -f deploy/k8s/pg-pvc.yaml +kubectl apply -f deploy/k8s/pg-statefulset.yaml +kubectl apply -f deploy/k8s/pg-service.yaml +``` + +| 参数 | 值 | 说明 | +| :--- | :--- | :--- | +| **镜像** | `postgres:13.16` | PostgreSQL 13.16 | +| **NodePort** | `30432` | 集群外访问端口 | +| **数据库** | `demo` | ConfigMap 中 `POSTGRES_DB` | +| **用户名** | `postgres` | ConfigMap 中 `POSTGRES_USER` | +| **密码** | `coslight` | ConfigMap `postgres-config` 中配置，生产环境迁移至 Secret | +| **存储** | `2Gi` | PVC `postgres-data` | + +##### 4.4.1 等待 Pod 就绪 + +```bash +kubectl wait --for=condition=ready pod -l app=postgres --timeout=120s +``` + +##### 4.4.2 初始化异步任务表 + +PostgreSQL 就绪后执行 1.4 节的建表 SQL，可通过以下方式进入容器执行： + +```bash +# 交互式 psql +kubectl exec -it $(kubectl get pod -l app=postgres -o jsonpath='{.items[0].metadata.name}') \ + -- psql -U postgres -d demo + +# 或将 SQL 文件通过管道一次性执行 +kubectl exec -i $(kubectl get pod -l app=postgres -o jsonpath='{.items[0].metadata.name}') \ + -- psql -U postgres -d demo < /path/to/init.sql +``` + +##### 4.4.3 状态检查 + +```bash +kubectl get pods -l app=postgres +kubectl logs -l app=postgres --tail=30 +``` + +##### 4.4.4 清理 + +```bash +kubectl delete -f deploy/k8s/pg-service.yaml \ + -f deploy/k8s/pg-statefulset.yaml \ + -f deploy/k8s/pg-pvc.yaml \ + -f deploy/k8s/pg-configmap.yaml +``` + +#### 4.5 部署 MongoDB + +```bash +kubectl apply -f deploy/k8s/mongodb-secret.yaml +kubectl apply -f deploy/k8s/mongodb-pvc.yaml +kubectl apply -f deploy/k8s/mongodb-statefulset.yaml +kubectl apply -f deploy/k8s/mongodb-service.yaml +``` + +| 参数 | 值 | 说明 | +| :--- | :--- | :--- | +| **镜像** | `mongo:7.0` | MongoDB 7.0 | +| **NodePort** | `30017` | 集群外访问端口 | +| **用户名** | `admin` | Root 管理员 | +| **密码** | `coslight` | Secret `mongodb-secret` 中配置，生产环境请替换强密码 | +| **存储** | `2Gi` | PVC `mongodb-data` | + +> **注意：** 密码存储在 `mongodb-secret.yaml` 的 `stringData` 中，生产环境应替换为强密码，并避免将明文密码提交至版本库。 + +##### 4.5.1 等待 Pod 就绪 + +```bash +kubectl wait --for=condition=ready pod -l app=mongodb --timeout=120s +``` + +##### 4.5.2 连接验证 + +```bash +kubectl exec -it $(kubectl get pod -l app=mongodb -o jsonpath='{.items[0].metadata.name}') \ + -- mongosh -u admin -p coslight --authenticationDatabase admin +``` + +##### 4.5.3 状态检查 + +```bash +kubectl get pods -l app=mongodb +kubectl logs -l app=mongodb --tail=30 +``` + +##### 4.5.4 清理 + +```bash +kubectl delete -f deploy/k8s/mongodb-service.yaml \ + -f deploy/k8s/mongodb-statefulset.yaml \ + -f deploy/k8s/mongodb-pvc.yaml \ + -f deploy/k8s/mongodb-secret.yaml +``` + ### 5\. 部署 ModelRT（Kubernetes） 所有资源部署在 `default` 命名空间，YAML 文件位于 `deploy/k8s/`。 @@ -852,7 +954,9 @@ Mac 本地端口 ──SSH隧道──▶ Ubuntu 宿主机 (192.168.1.101) #### 7.2 建立隧道 ```bash -ssh -L 5671:192.168.49.2:30671 \ +ssh -L 5432:192.168.49.2:30432 \ + -L 27017:192.168.49.2:30017 \ + -L 5671:192.168.49.2:30671 \ -L 15671:192.168.49.2:31671 \ -L 6379:192.168.49.2:30001 \ -L 4318:192.168.49.2:31318 \ @@ -866,6 +970,8 @@ ssh -L 5671:192.168.49.2:30671 \ ```bash ssh -fN \ + -L 5432:192.168.49.2:30432 \ + -L 27017:192.168.49.2:30017 \ -L 5671:192.168.49.2:30671 \ -L 15671:192.168.49.2:31671 \ -L 6379:192.168.49.2:30001 \ @@ -880,6 +986,8 @@ ssh -fN \ | Mac 本地端口 | Minikube NodePort | 服务 | 说明 | | :--- | :--- | :--- | :--- | +| `5432` | `30432` | PostgreSQL | 数据库连接 `localhost:5432` | +| `27017` | `30017` | MongoDB | 数据库连接 `localhost:27017` | | `5671` | `30671` | RabbitMQ AMQP | ModelRT / EventRT 消息队列连接 | | `15671` | `31671` | RabbitMQ Management | RabbitMQ 管理界面 `http://localhost:15671` | | `6379` | `30001` | Redis | 分布式锁 / 数据存储 | diff --git a/deploy/k8s/mongodb-pvc.yaml b/deploy/k8s/mongodb-pvc.yaml new file mode 100644 index 0000000..d009b0a --- /dev/null +++ b/deploy/k8s/mongodb-pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongodb-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/deploy/k8s/mongodb-secret.yaml b/deploy/k8s/mongodb-secret.yaml new file mode 100644 index 0000000..53363f2 --- /dev/null +++ b/deploy/k8s/mongodb-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: mongodb-secret +type: Opaque +stringData: + MONGO_INITDB_ROOT_USERNAME: admin + MONGO_INITDB_ROOT_PASSWORD: coslight diff --git a/deploy/k8s/mongodb-service.yaml b/deploy/k8s/mongodb-service.yaml new file mode 100644 index 0000000..daf946a --- /dev/null +++ b/deploy/k8s/mongodb-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: mongodb + labels: + app: mongodb +spec: + type: NodePort + selector: + app: mongodb + ports: + - name: mongodb + port: 27017 + targetPort: 27017 + nodePort: 30017 diff --git a/deploy/k8s/mongodb-statefulset.yaml b/deploy/k8s/mongodb-statefulset.yaml new file mode 100644 index 0000000..8d1de21 --- /dev/null +++ b/deploy/k8s/mongodb-statefulset.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mongodb + labels: + app: mongodb +spec: + serviceName: mongodb + replicas: 1 + selector: + matchLabels: + app: mongodb + template: + metadata: + labels: + app: mongodb + spec: + containers: + - name: mongodb + image: mongo:7.0 + imagePullPolicy: IfNotPresent + ports: + - name: mongodb + containerPort: 27017 + envFrom: + - secretRef: + name: mongodb-secret + volumeMounts: + - name: mongodb-data + mountPath: /data/db + readinessProbe: + exec: + command: + - mongosh + - --eval + - "db.adminCommand('ping')" + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 12 + livenessProbe: + exec: + command: + - mongosh + - --eval + - "db.adminCommand('ping')" + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumes: + - name: mongodb-data + persistentVolumeClaim: + claimName: mongodb-data diff --git a/deploy/redis-test-data/real-time-subpull/sub_data_injection.go b/deploy/redis-test-data/real-time-subpull/sub_data_injection.go index a15cef6..4c0728b 100644 --- a/deploy/redis-test-data/real-time-subpull/sub_data_injection.go +++ b/deploy/redis-test-data/real-time-subpull/sub_data_injection.go @@ -129,9 +129,9 @@ func generateOutlierSegments(totalSize, minLength, maxLength, count int, distrib segments := make([]OutlierSegment, 0, count) usedPositions := make(map[int]bool) - for i := 0; i < count; i++ { + for range count { // 尝试多次寻找合适的位置 - for attempt := 0; attempt < 10; attempt++ { + for range 10 { length := rand.Intn(maxLength-minLength+1) + minLength start := rand.Intn(totalSize - length) diff --git a/diagram/hash_test.go b/diagram/hash_test.go index 5ba91fd..5b2053f 100644 --- a/diagram/hash_test.go +++ b/diagram/hash_test.go @@ -18,7 +18,7 @@ func TestHMSet(t *testing.T) { PoolSize: 50, DialTimeout: 10 * time.Second, }) - params := map[string]interface{}{ + params := map[string]any{ "field1": "Hello1", "field2": "World1", "field3": 11, diff --git a/diagram/multi_branch_tree.go b/diagram/multi_branch_tree.go index d88393b..d61e385 100644 --- a/diagram/multi_branch_tree.go +++ b/diagram/multi_branch_tree.go @@ -53,7 +53,7 @@ func (n *MultiBranchTreeNode) FindNodeByID(id uuid.UUID) *MultiBranchTreeNode { } func (n *MultiBranchTreeNode) PrintTree(level int) { - for i := 0; i < level; i++ { + for range level { fmt.Print(" ") } diff --git a/diagram/redis_hash.go b/diagram/redis_hash.go index 9ef0f22..2382828 100644 --- a/diagram/redis_hash.go +++ b/diagram/redis_hash.go @@ -29,7 +29,7 @@ func NewRedisHash(ctx context.Context, hashKey string, lockLeaseTime uint64, nee } // SetRedisHashByMap define func of set redis hash by map struct -func (rh *RedisHash) SetRedisHashByMap(fields map[string]interface{}) error { +func (rh *RedisHash) SetRedisHashByMap(fields map[string]any) error { err := rh.rwLocker.WLock(rh.ctx) if err != nil { logger.Error(rh.ctx, "lock wLock by hash_key failed", "hash_key", rh.hashKey, "error", err) @@ -46,7 +46,7 @@ func (rh *RedisHash) SetRedisHashByMap(fields map[string]interface{}) error { } // SetRedisHashByKV define func of set redis hash by kv struct -func (rh *RedisHash) SetRedisHashByKV(field string, value interface{}) error { +func (rh *RedisHash) SetRedisHashByKV(field string, value any) error { err := rh.rwLocker.WLock(rh.ctx) if err != nil { logger.Error(rh.ctx, "lock wLock by hash_key failed", "hash_key", rh.hashKey, "error", err) diff --git a/diagram/redis_string.go b/diagram/redis_string.go index 94b7bd9..130a89a 100644 --- a/diagram/redis_string.go +++ b/diagram/redis_string.go @@ -46,7 +46,7 @@ func (rs *RedisString) Get(stringKey string) (string, error) { } // Set define func of set the value of key -func (rs *RedisString) Set(stringKey string, value interface{}) error { +func (rs *RedisString) Set(stringKey string, value any) error { err := rs.rwLocker.WLock(rs.ctx) if err != nil { logger.Error(rs.ctx, "lock wLock by stringKey failed", "string_key", stringKey, "error", err) diff --git a/diagram/redis_zset.go b/diagram/redis_zset.go index 549d28b..d350b4f 100644 --- a/diagram/redis_zset.go +++ b/diagram/redis_zset.go @@ -30,7 +30,7 @@ func NewRedisZSet(ctx context.Context, key string, lockLeaseTime uint64, needRef } // ZADD define func of add redis zset by members -func (rs *RedisZSet) ZADD(setKey string, score float64, member interface{}) error { +func (rs *RedisZSet) ZADD(setKey string, score float64, member any) error { err := rs.rwLocker.WLock(rs.ctx) if err != nil { logger.Error(rs.ctx, "lock wLock by setKey failed", "set_key", setKey, "error", err) diff --git a/diagram/topologic_set.go b/diagram/topologic_set.go index 607584f..ccc17a9 100644 --- a/diagram/topologic_set.go +++ b/diagram/topologic_set.go @@ -12,7 +12,7 @@ var graphOverview sync.Map // PrintGrapMap define func of print circuit diagram topologic info data func PrintGrapMap() { - graphOverview.Range(func(key, value interface{}) bool { + graphOverview.Range(func(key, value any) bool { fmt.Println(key, value) return true }) diff --git a/handler/anchor_point_replace.go b/handler/anchor_point_replace.go index 877c877..9bdf0ab 100644 --- a/handler/anchor_point_replace.go +++ b/handler/anchor_point_replace.go @@ -68,7 +68,7 @@ func ComponentAnchorReplaceHandler(c *gin.Context) { resp := network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": request.UUID, }, } diff --git a/handler/async_task_create_handler.go b/handler/async_task_create_handler.go index 30b8531..d1ff912 100644 --- a/handler/async_task_create_handler.go +++ b/handler/async_task_create_handler.go @@ -127,7 +127,7 @@ func validatePerformanceAnalysisParams(params map[string]any) bool { // Check required parameters for performance analysis if componentIDs, ok := params["component_ids"]; !ok { return false - } else if ids, isSlice := componentIDs.([]interface{}); !isSlice || len(ids) == 0 { + } else if ids, isSlice := componentIDs.([]any); !isSlice || len(ids) == 0 { return false } return true diff --git a/handler/attr_delete.go b/handler/attr_delete.go index fedb49a..783c01f 100644 --- a/handler/attr_delete.go +++ b/handler/attr_delete.go @@ -41,7 +41,7 @@ func AttrDeleteHandler(c *gin.Context) { c.JSON(http.StatusOK, network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{"attr_token": request.AttrToken}, + Payload: map[string]any{"attr_token": request.AttrToken}, }) return } @@ -49,7 +49,7 @@ func AttrDeleteHandler(c *gin.Context) { c.JSON(http.StatusOK, network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "attr_token": request.AttrToken, }, }) diff --git a/handler/attr_load.go b/handler/attr_load.go index 0c07744..3591910 100644 --- a/handler/attr_load.go +++ b/handler/attr_load.go @@ -46,7 +46,7 @@ func AttrGetHandler(c *gin.Context) { c.JSON(http.StatusOK, network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{"attr_token": request.AttrToken}, + Payload: map[string]any{"attr_token": request.AttrToken}, }) return } @@ -59,7 +59,7 @@ func AttrGetHandler(c *gin.Context) { c.JSON(http.StatusOK, network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "attr_token": request.AttrToken, "attr_value": attrValue, }, diff --git a/handler/attr_update.go b/handler/attr_update.go index 8a57599..226631f 100644 --- a/handler/attr_update.go +++ b/handler/attr_update.go @@ -43,7 +43,7 @@ func AttrSetHandler(c *gin.Context) { c.JSON(http.StatusOK, network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{"attr_token": request.AttrToken}, + Payload: map[string]any{"attr_token": request.AttrToken}, }) return } @@ -51,7 +51,7 @@ func AttrSetHandler(c *gin.Context) { c.JSON(http.StatusOK, network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "attr_token": request.AttrToken, }, }) diff --git a/handler/circuit_diagram_create.go b/handler/circuit_diagram_create.go index 536151d..854e0c8 100644 --- a/handler/circuit_diagram_create.go +++ b/handler/circuit_diagram_create.go @@ -37,7 +37,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, }, } @@ -65,7 +65,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicLink, }, } @@ -89,7 +89,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_infos": topologicCreateInfos, }, } @@ -111,7 +111,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "component_infos": request.ComponentInfos, }, } @@ -130,7 +130,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": info.UUID, "component_params": info.Params, }, @@ -152,7 +152,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) { resp := network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, }, } diff --git a/handler/circuit_diagram_delete.go b/handler/circuit_diagram_delete.go index a691679..2025ad0 100644 --- a/handler/circuit_diagram_delete.go +++ b/handler/circuit_diagram_delete.go @@ -42,7 +42,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, }, } @@ -70,7 +70,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicLink, }, } @@ -95,7 +95,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicDelInfo, }, } @@ -112,7 +112,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicDelInfo, }, } @@ -138,7 +138,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": componentInfo.UUID, }, } @@ -162,7 +162,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": componentInfo.UUID, }, } @@ -184,7 +184,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": componentInfo.UUID, }, } @@ -205,7 +205,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, }, } diff --git a/handler/circuit_diagram_load.go b/handler/circuit_diagram_load.go index 8f8c71c..7ad390b 100644 --- a/handler/circuit_diagram_load.go +++ b/handler/circuit_diagram_load.go @@ -33,7 +33,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": pageID, }, } @@ -48,14 +48,14 @@ func CircuitDiagramLoadHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": pageID, }, } c.JSON(http.StatusOK, resp) return } - payload := make(map[string]interface{}) + payload := make(map[string]any) payload["root_vertex"] = topologicInfo.RootVertex payload["topologic"] = topologicInfo.VerticeLinks @@ -69,7 +69,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": componentUUID, }, } @@ -84,7 +84,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": componentUUID, }, } @@ -103,7 +103,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": topologicInfo.RootVertex, }, } @@ -118,7 +118,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": rootVertexUUID, }, } diff --git a/handler/circuit_diagram_update.go b/handler/circuit_diagram_update.go index d0181e1..33bb46b 100644 --- a/handler/circuit_diagram_update.go +++ b/handler/circuit_diagram_update.go @@ -35,7 +35,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, }, } @@ -52,7 +52,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicLink, }, } @@ -75,7 +75,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicChangeInfo, }, } @@ -92,7 +92,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicChangeInfo, }, } @@ -109,7 +109,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, "component_info": request.ComponentInfos, }, @@ -129,7 +129,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": info.UUID, "component_params": info.Params, }, @@ -152,7 +152,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, }, } diff --git a/handler/history_data_query.go b/handler/history_data_query.go index d8718e1..f9df2a2 100644 --- a/handler/history_data_query.go +++ b/handler/history_data_query.go @@ -50,7 +50,7 @@ func QueryHistoryDataHandler(c *gin.Context) { resp := network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "events": events, }, } diff --git a/handler/real_time_data_receive.go b/handler/real_time_data_receive.go index 77ab71f..de09a8a 100644 --- a/handler/real_time_data_receive.go +++ b/handler/real_time_data_receive.go @@ -64,7 +64,7 @@ func RealTimeDataReceivehandler(c *gin.Context) { realtimedata.RealTimeDataChan <- request - payload := map[string]interface{}{ + payload := map[string]any{ "component_uuid": request.PayLoad.ComponentUUID, "point": request.PayLoad.Point, } @@ -82,8 +82,8 @@ func RealTimeDataReceivehandler(c *gin.Context) { } } -func processResponse(code int64, msg string, payload map[string]interface{}) []byte { - resp := map[string]interface{}{ +func processResponse(code int64, msg string, payload map[string]any) []byte { + resp := map[string]any{ "code": code, "msg": msg, "payload": payload, diff --git a/logger/loki_syncer.go b/logger/loki_syncer.go index 3e3f319..332ddcf 100644 --- a/logger/loki_syncer.go +++ b/logger/loki_syncer.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "fmt" + "maps" "net/http" "os" "strconv" @@ -39,9 +40,7 @@ type lokiSyncer struct { func newLokiSyncer(lCfg config.LokiConfig) *lokiSyncer { // always tag development logs with env=development; caller-supplied labels override if needed labels := map[string]string{"env": "development"} - for k, v := range lCfg.Labels { - labels[k] = v - } + maps.Copy(labels, lCfg.Labels) ls := &lokiSyncer{ endpoint: lCfg.Endpoint + "/loki/api/v1/push", labels: labels, diff --git a/middleware/trace.go b/middleware/trace.go index 65c1ee2..c113d20 100644 --- a/middleware/trace.go +++ b/middleware/trace.go @@ -139,7 +139,7 @@ func LogAccess() gin.HandlerFunc { } } -func accessLog(c *gin.Context, accessType string, dur time.Duration, body []byte, dataOut interface{}) { +func accessLog(c *gin.Context, accessType string, dur time.Duration, body []byte, dataOut any) { req := c.Request bodyStr := string(body) query := req.URL.RawQuery diff --git a/model/attribute_model.go b/model/attribute_model.go index ff7d55e..601d156 100644 --- a/model/attribute_model.go +++ b/model/attribute_model.go @@ -11,7 +11,7 @@ type AttrModelInterface interface { GetZoneInfo() *orm.Zone GetStationInfo() *orm.Station GetComponentInfo() *orm.Component - GetAttrValue() interface{} // New method to get the attribute value + GetAttrValue() any // New method to get the attribute value IsLocal() bool } diff --git a/network/api_endpoint.go b/network/api_endpoint.go index 6f15665..cac8d2a 100644 --- a/network/api_endpoint.go +++ b/network/api_endpoint.go @@ -77,7 +77,7 @@ func PollAPIEndpoints(endpoint APIEndpoint) ([]float64, error) { } dataLen := len(realDataJSON.Get("data").MustArray()) - for i := 0; i < dataLen; i++ { + for i := range dataLen { valueSlice = append(valueSlice, realDataJSON.Get("data").GetIndex(i).Get("value").MustFloat64()) } return valueSlice, nil diff --git a/network/async_task_request.go b/network/async_task_request.go index bf6a55a..8982ef6 100644 --- a/network/async_task_request.go +++ b/network/async_task_request.go @@ -13,7 +13,7 @@ type AsyncTaskCreateRequest struct { // enum: TOPOLOGY_ANALYSIS, PERFORMANCE_ANALYSIS, EVENT_ANALYSIS, BATCH_IMPORT TaskType string `json:"task_type" example:"TOPOLOGY_ANALYSIS" description:"异步任务类型"` // required: true - Params map[string]interface{} `json:"params" swaggertype:"object" description:"任务参数，根据任务类型不同而不同"` + Params map[string]any `json:"params" swaggertype:"object" description:"任务参数，根据任务类型不同而不同"` } // AsyncTaskCreateResponse defines the response structure for creating an asynchronous task @@ -29,16 +29,16 @@ type AsyncTaskResultQueryRequest struct { // AsyncTaskResult defines the structure for a single task result type AsyncTaskResult struct { - TaskID uuid.UUID `json:"task_id" example:"123e4567-e89b-12d3-a456-426614174000" description:"任务唯一标识符"` - TaskType string `json:"task_type" example:"TOPOLOGY_ANALYSIS" description:"任务类型"` - Status string `json:"status" example:"COMPLETED" description:"任务状态：SUBMITTED, RUNNING, COMPLETED, FAILED"` - Progress *int `json:"progress,omitempty" example:"65" description:"任务进度(0-100)，仅当状态为RUNNING时返回"` - CreatedAt int64 `json:"created_at" example:"1741846200" description:"任务创建时间戳"` - FinishedAt *int64 `json:"finished_at,omitempty" example:"1741846205" description:"任务完成时间戳，仅当状态为COMPLETED或FAILED时返回"` - Result map[string]interface{} `json:"result,omitempty" swaggertype:"object" description:"任务结果，仅当状态为COMPLETED时返回"` - ErrorCode *int `json:"error_code,omitempty" example:"400102" description:"错误码，仅当状态为FAILED时返回"` - ErrorMessage *string `json:"error_message,omitempty" example:"Component UUID not found" description:"错误信息，仅当状态为FAILED时返回"` - ErrorDetail map[string]interface{} `json:"error_detail,omitempty" swaggertype:"object" description:"错误详情，仅当状态为FAILED时返回"` + TaskID uuid.UUID `json:"task_id" example:"123e4567-e89b-12d3-a456-426614174000" description:"任务唯一标识符"` + TaskType string `json:"task_type" example:"TOPOLOGY_ANALYSIS" description:"任务类型"` + Status string `json:"status" example:"COMPLETED" description:"任务状态：SUBMITTED, RUNNING, COMPLETED, FAILED"` + Progress *int `json:"progress,omitempty" example:"65" description:"任务进度(0-100)，仅当状态为RUNNING时返回"` + CreatedAt int64 `json:"created_at" example:"1741846200" description:"任务创建时间戳"` + FinishedAt *int64 `json:"finished_at,omitempty" example:"1741846205" description:"任务完成时间戳，仅当状态为COMPLETED或FAILED时返回"` + Result map[string]any `json:"result,omitempty" swaggertype:"object" description:"任务结果，仅当状态为COMPLETED时返回"` + ErrorCode *int `json:"error_code,omitempty" example:"400102" description:"错误码，仅当状态为FAILED时返回"` + ErrorMessage *string `json:"error_message,omitempty" example:"Component UUID not found" description:"错误信息，仅当状态为FAILED时返回"` + ErrorDetail map[string]any `json:"error_detail,omitempty" swaggertype:"object" description:"错误详情，仅当状态为FAILED时返回"` } // AsyncTaskResultQueryResponse defines the response structure for querying task results diff --git a/network/attr_request.go b/network/attr_request.go index fad5edd..624f5f9 100644 --- a/network/attr_request.go +++ b/network/attr_request.go @@ -8,8 +8,8 @@ type AttrGetRequest struct { // AttrSetRequest defines the request payload for setting an attribute type AttrSetRequest struct { - AttrToken string `json:"attr_token"` - AttrValue interface{} `json:"attr_value"` + AttrToken string `json:"attr_token"` + AttrValue any `json:"attr_value"` } // AttrDeleteRequest defines the request payload for deleting an attribute diff --git a/network/request_convert.go b/network/request_convert.go index 85c9fe0..11ca2d3 100644 --- a/network/request_convert.go +++ b/network/request_convert.go @@ -8,7 +8,7 @@ import ( ) // ConvertAnyComponentInfosToComponents define convert any component request info to component struct -func ConvertAnyComponentInfosToComponents(anyInfo interface{}) (*orm.Component, error) { +func ConvertAnyComponentInfosToComponents(anyInfo any) (*orm.Component, error) { switch info := anyInfo.(type) { case ComponentCreateInfo: return ConvertComponentCreateInfosToComponents(info) diff --git a/pool/concurrency_anchor_parse.go b/pool/concurrency_anchor_parse.go index 85fa225..1e82723 100644 --- a/pool/concurrency_anchor_parse.go +++ b/pool/concurrency_anchor_parse.go @@ -28,7 +28,7 @@ func AnchorPoolInit(concurrentQuantity int) (pool *ants.PoolWithFunc, err error) } // AnchorFunc defines func that process the real time data of component anchor params -var AnchorFunc = func(poolConfig interface{}) { +var AnchorFunc = func(poolConfig any) { var firstStart bool alertManager := alert.GetAlertMangerInstance() diff --git a/pool/concurrency_model_parse.go b/pool/concurrency_model_parse.go index 1fe8718..0872fb0 100644 --- a/pool/concurrency_model_parse.go +++ b/pool/concurrency_model_parse.go @@ -12,7 +12,7 @@ import ( ) // ParseFunc defines func that parses the model data from postgres -var ParseFunc = func(parseConfig interface{}) { +var ParseFunc = func(parseConfig any) { modelParseConfig, ok := parseConfig.(config.ModelParseConfig) if !ok { logger.Error(modelParseConfig.Ctx, "conversion model parse config type failed") diff --git a/real-time-data/cache.go b/real-time-data/cache.go index 70a2d03..585a447 100644 --- a/real-time-data/cache.go +++ b/real-time-data/cache.go @@ -10,7 +10,7 @@ import ( // DataItem define structure for storing data, insertion time, and last access time type DataItem struct { - Data interface{} + Data any InsertTime time.Time LastAccess time.Time Index int @@ -38,14 +38,14 @@ func (pq priorityQueue) Swap(i, j int) { pq[j].item.Index = j } -func (pq *priorityQueue) Push(x interface{}) { +func (pq *priorityQueue) Push(x any) { n := len(*pq) queueItem := x.(*priorityQueueItem) queueItem.item.Index = n *pq = append(*pq, queueItem) } -func (pq *priorityQueue) Pop() interface{} { +func (pq *priorityQueue) Pop() any { old := *pq n := len(old) queueItem := old[n-1] @@ -65,7 +65,7 @@ func (pq *priorityQueue) update(item *DataItem, newPrio int64) { type TimeCache struct { mu sync.Mutex capacity int - items map[interface{}]*DataItem + items map[any]*DataItem pq priorityQueue } @@ -73,13 +73,13 @@ type TimeCache struct { func NewTimeCache(capacity int) *TimeCache { return &TimeCache{ capacity: capacity, - items: make(map[interface{}]*DataItem), + items: make(map[any]*DataItem), pq: make(priorityQueue, 0, capacity), } } // Add 添加一个新项到缓存中 -func (tc *TimeCache) Add(data interface{}) { +func (tc *TimeCache) Add(data any) { tc.mu.Lock() defer tc.mu.Unlock() diff --git a/task/base_task.go b/task/base_task.go index c49b897..9d29a1a 100644 --- a/task/base_task.go +++ b/task/base_task.go @@ -13,8 +13,8 @@ import ( type Params interface { Validate() error GetType() UnifiedTaskType - ToMap() map[string]interface{} - FromMap(params map[string]interface{}) error + ToMap() map[string]any + FromMap(params map[string]any) error } // BaseTask provides common functionality for all task implementations diff --git a/task/retry_manager.go b/task/retry_manager.go index be70c3a..2ce83b3 100644 --- a/task/retry_manager.go +++ b/task/retry_manager.go @@ -22,10 +22,10 @@ type RetryStrategy interface { // ExponentialBackoffRetry implements exponential backoff with jitter retry strategy type ExponentialBackoffRetry struct { - MaxRetries int - InitialDelay time.Duration - MaxDelay time.Duration - RandomFactor float64 // Jitter factor to avoid thundering herd problem + MaxRetries int + InitialDelay time.Duration + MaxDelay time.Duration + RandomFactor float64 // Jitter factor to avoid thundering herd problem } // NewExponentialBackoffRetry creates a new exponential backoff retry strategy @@ -67,12 +67,9 @@ func (s *ExponentialBackoffRetry) ShouldRetry(ctx context.Context, taskID string } // Calculate exponential backoff: initialDelay * 2^retryCount - delay := s.InitialDelay * time.Duration(math.Pow(2, float64(retryCount))) - - // Apply maximum delay cap - if delay > s.MaxDelay { - delay = s.MaxDelay - } + delay := min( + // Apply maximum delay cap + s.InitialDelay*time.Duration(math.Pow(2, float64(retryCount))), s.MaxDelay) // Add jitter to avoid thundering herd if s.RandomFactor > 0 { @@ -178,10 +175,10 @@ func (s *NoRetryStrategy) GetMaxRetries() int { // DefaultRetryStrategy returns the default retry strategy (exponential backoff) func DefaultRetryStrategy() RetryStrategy { return NewExponentialBackoffRetry( - constants.TaskRetryMaxDefault, // max retries - constants.TaskRetryInitialDelayDefault, // initial delay - constants.TaskRetryMaxDelayDefault, // max delay - constants.TaskRetryRandomFactorDefault, // random factor (10% jitter) + constants.TaskRetryMaxDefault, // max retries + constants.TaskRetryInitialDelayDefault, // initial delay + constants.TaskRetryMaxDelayDefault, // max delay + constants.TaskRetryRandomFactorDefault, // random factor (10% jitter) ) } diff --git a/task/test_task.go b/task/test_task.go index 580d1a2..14edb14 100644 --- a/task/test_task.go +++ b/task/test_task.go @@ -45,15 +45,15 @@ func (p *TestTaskParams) GetType() UnifiedTaskType { } // ToMap converts parameters to map for database storage -func (p *TestTaskParams) ToMap() map[string]interface{} { - return map[string]interface{}{ +func (p *TestTaskParams) ToMap() map[string]any { + return map[string]any{ "sleep_duration": p.SleepDuration, "message": p.Message, } } // FromMap populates parameters from map (for database retrieval) -func (p *TestTaskParams) FromMap(params map[string]interface{}) error { +func (p *TestTaskParams) FromMap(params map[string]any) error { if v, ok := params["sleep_duration"]; ok { if duration, isFloat := v.(float64); isFloat { p.SleepDuration = int(duration) @@ -103,7 +103,7 @@ func (t *TestTask) Execute(ctx context.Context, taskID uuid.UUID, db *gorm.DB) e time.Sleep(sleepDuration) // Build result - result := map[string]interface{}{ + result := map[string]any{ "status": "completed", "sleep_duration": params.SleepDuration, "message": params.Message, diff --git a/task/worker.go b/task/worker.go index 1febca4..d60c1a3 100644 --- a/task/worker.go +++ b/task/worker.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "maps" "sync" "time" @@ -565,24 +566,16 @@ func (w *TaskWorker) GetMetrics() *WorkerMetrics { // Deep copy maps to avoid data races tasksProcessedCopy := make(map[TaskType]int64) - for k, v := range w.metrics.TasksProcessed { - tasksProcessedCopy[k] = v - } + maps.Copy(tasksProcessedCopy, w.metrics.TasksProcessed) tasksFailedCopy := make(map[TaskType]int64) - for k, v := range w.metrics.TasksFailed { - tasksFailedCopy[k] = v - } + maps.Copy(tasksFailedCopy, w.metrics.TasksFailed) tasksSuccessCopy := make(map[TaskType]int64) - for k, v := range w.metrics.TasksSuccess { - tasksSuccessCopy[k] = v - } + maps.Copy(tasksSuccessCopy, w.metrics.TasksSuccess) processingTimeCopy := make(map[TaskType]time.Duration) - for k, v := range w.metrics.ProcessingTime { - processingTimeCopy[k] = v - } + maps.Copy(processingTimeCopy, w.metrics.ProcessingTime) // Create a copy without the mutex to avoid copylocks warning return &WorkerMetrics{